From 9f8fb60f8a3db785204e882724d10e14609f70c3 Mon Sep 17 00:00:00 2001 From: Tony Theodore Date: Mon, 13 Nov 2017 19:56:53 +1100 Subject: [PATCH 1/2] print-deps-for-build-pkg: sort and strip output for repeatable comparison --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index ac3af606..e1c07bbf 100644 --- a/Makefile +++ b/Makefile @@ -736,13 +736,13 @@ show-upstream-deps-%: # print first level pkg deps for use in build-pkg.lua .PHONY: print-deps-for-build-pkg print-deps-for-build-pkg: - $(foreach TARGET,$(MXE_TARGETS), \ + $(foreach TARGET,$(sort $(MXE_TARGETS)), \ $(foreach PKG,$(sort $($(TARGET)_PKGS)), \ - $(info for-build-pkg $(TARGET)~$(PKG) \ + $(info $(strip for-build-pkg $(TARGET)~$(PKG) \ $(subst $(space),-,$($(PKG)_VERSION)) \ $(addprefix $(TARGET)~,$(value $(call LOOKUP_PKG_RULE,$(PKG),DEPS,$(TARGET)))) \ $(addprefix $(TARGET)~,$(if $(call set_is_not_member,$(PKG),$(MXE_CONF_PKGS)),$(MXE_CONF_PKGS))) \ - $(and $($(TARGET)_DEPS),$(addprefix $($(TARGET)_DEPS)~,$($($(TARGET)_DEPS)_PKGS)))))) + $(and $($(TARGET)_DEPS),$(addprefix $($(TARGET)_DEPS)~,$($($(TARGET)_DEPS)_PKGS))))))) @echo -n BUILD_PKG_TMP_FILES := *-*.list mxe-*.tar.xz mxe-*.deb* wheezy jessie From a3624b48b1600dcae5fc9df7d436269ecb90b090 Mon Sep 17 00:00:00 2001 From: Tony Theodore Date: Mon, 13 Nov 2017 20:00:37 +1100 Subject: [PATCH 2/2] Makefile: speedup LOOKUP_PKG_RULE LOOKUP_PKG_RULE is called many times and spends a lot of time searching for rules when the default rule is the most common. This simply avoids exhaustive searching since we can determine beforehand if there are multiple rules. Also drops some of the gmsl functions if favour of normal make variables. First commit adds sort/strip so print-deps-for-build-pkg output can be easily compared. ``` $ git checkout HEAD~1 $ time make print-deps-for-build-pkg \ MXE_TARGETS="`echo {i686-w64-mingw32,x86_64-w64-mingw32}.{static,shared}`"> orig real 0m17.564s user 0m16.410s sys 0m0.783s $ git checkout speedup $ time make print-deps-for-build-pkg \ MXE_TARGETS="`echo {i686-w64-mingw32,x86_64-w64-mingw32}.{static,shared}`"> new real 0m4.036s user 0m3.624s sys 0m0.326s $ diff -u orig new $ ``` --- Makefile | 75 ++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index e1c07bbf..5a2dd772 100644 --- a/Makefile +++ b/Makefile @@ -366,31 +366,6 @@ LIST_NMIN = $(shell echo '$(strip $(1))' | tr ' ' '\n' | sort -n | head -1) NPROCS := $(shell nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1) JOBS := $(call LIST_NMIN, $(DEFAULT_MAX_JOBS) $(NPROCS)) -# cache some target string manipulation functions -# `memoize` and `uc` from gmsl -_CHOP_TARGET = $(call merge,.,$(call chop,$(call split,.,$(1)))) -CHOP_TARGET = $(call memoize,_CHOP_TARGET,$(1)) -_UC_LIB_TYPE = $(call uc,$(word 2,$(subst ., ,$(1)))) -UC_LIB_TYPE = $(call memoize,_UC_LIB_TYPE,$(1)) - -# finds a package build rule or deps by truncating the target elements -# $(call LOOKUP_PKG_RULE, package, rule type ie. BUILD|DEPS|FILE, target,[lib type, original target to cache]) -# returns variable name for use with $(value) -# -# caches result with gmsl associative arrays (`get` and `set` functions) -# since `memoize` only works with single argument -LOOKUP_PKG_RULE = $(strip \ - $(or $(call get,LOOKUP_PKG_RULE_,$(1)_$(2)_$(or $(5),$(3))),\ - $(if $(findstring undefined, $(flavor $(1)_$(2)_$(3))),\ - $(if $(3),\ - $(call LOOKUP_PKG_RULE,$(1),$(2),$(call CHOP_TARGET,$(3)),$(or $(4),$(call UC_LIB_TYPE,$(3))),$(or $(5),$(3))),\ - $(if $(4),\ - $(call LOOKUP_PKG_RULE,$(1),$(2),$(4),,$(5)),\ - $(call set,LOOKUP_PKG_RULE_,$(1)_$(2)_$(5),$(1)_$(2))\ - $(1)_$(2))),\ - $(call set,LOOKUP_PKG_RULE_,$(1)_$(2)_$(or $(5),$(3)),$(1)_$(2)_$(3))\ - $(1)_$(2)_$(3)))) - # Core packages. override MXE_PLUGIN_DIRS := $(realpath $(TOP_DIR)/src) $(MXE_PLUGIN_DIRS) @@ -466,6 +441,46 @@ $(foreach TARGET,$(CROSS_TARGETS),\ # always add $(BUILD) to our targets override MXE_TARGETS := $(CROSS_TARGETS) $(BUILD) +# cache some target string manipulation functions with normal make variables +CHOP_TARGETS = \ + $(if $(1),\ + $(eval CHOPPED := $(call merge,.,$(call chop,$(call split,.,$(1)))))\ + $(eval $(1)_CHOPPED := $(CHOPPED))\ + $(call CHOP_TARGETS,$(CHOPPED))) + +$(foreach TARGET,$(MXE_TARGETS),\ + $(call CHOP_TARGETS,$(TARGET))\ + $(eval $(TARGET)_UC_LIB_TYPE := $(if $(findstring shared,$(TARGET)),SHARED,STATIC))) + +# finds a package rule defintion +RULE_TYPES := BUILD DEPS FILE MESSAGE URL +# by truncating the target elements then looking for STAIC|SHARED rules: +# +# foo_BUILD_i686-w64-mingw32.static.posix.dw2 +# foo_BUILD_i686-w64-mingw32.static.posix +# foo_BUILD_i686-w64-mingw32.static +# foo_BUILD_i686-w64-mingw32 +# foo_BUILD_SHARED +# foo_BUILD + +# return the pre-populated rule if defined +LOOKUP_PKG_RULE = $(or $(LOOKUP_PKG_RULE_$(1)_$(2)_$(3)),$(1)_$(2)) + +# $(call _LOOKUP_PKG_RULE, package, rule type, target [, lib type]) +# returns variable name for use with $(value). PKG_RULE below will +# populate LOOKUP_PKG_RULE_* variables for rules that require lookups +_LOOKUP_PKG_RULE = $(strip \ + $(if $(findstring undefined, $(flavor $(PKG)_$(RULE)_$(3))),\ + $(if $(3),\ + $(call _LOOKUP_PKG_RULE,$(PKG),$(RULE),$($(3)_CHOPPED),$(or $(4),$($(3)_UC_LIB_TYPE)))\ + $(else),\ + $(if $(4),\ + $(call _LOOKUP_PKG_RULE,$(PKG),$(RULE),$(4))\ + $(else),\ + $(PKG)_$(RULE)))\ + $(else),\ + $(PKG)_$(RULE)_$(3))) + # set column widths for build status messages PKG_COL_WIDTH := $(call plus,2,$(call LIST_NMAX, $(sort $(call map, strlen, $(PKGS))))) MAX_TARGET_WIDTH := $(call LIST_NMAX, $(sort $(call map, strlen, $(MXE_TARGETS)))) @@ -540,6 +555,12 @@ prepare-pkg-source-$(1): download-only-$(1) rm -rf '$(2)' mkdir -p '$(2)' $$(call PREPARE_PKG_SOURCE,$(1),$(2)) + +# populate LOOKUP_PKG_RULE_ variables where there are multiple defined +$(foreach RULE,$(RULE_TYPES),\ + $(if $(filter-out %_URL_2,$(filter-out $(PKG)_$(RULE),$(filter $(PKG)_$(RULE)%,$(.VARIABLES)))),\ + $(foreach TARGET,$(MXE_TARGETS),\ + $(eval LOOKUP_PKG_RULE_$(PKG)_$(RULE)_$(TARGET) := $(call _LOOKUP_PKG_RULE,$(PKG),$(RULE),$(TARGET)))))) endef $(foreach PKG,$(PKGS),$(eval $(call PKG_RULE,$(PKG),$(call TMP_DIR,$(PKG))))) @@ -691,9 +712,9 @@ WALK_DOWNSTREAM = \ $(strip \ $(foreach PKG,$(PKGS),\ $(call SET_CLEAR,PKGS_VISITED)\ - $(eval $(PKG)_DEPS_ALL := $(call WALK_UPSTREAM,$(PKG))))\ + $(eval ALL_$(PKG)_DEPS := $(call WALK_UPSTREAM,$(PKG))))\ $(foreach PKG,$(PKGS),\ - $(if $(filter $(1),$($(PKG)_DEPS_ALL)),$(PKG)))) + $(if $(filter $(1),$(ALL_$(PKG)_DEPS)),$(PKG)))) # EXCLUDE_PKGS can be a list of pkgs and/or wildcards RECURSIVELY_EXCLUDED_PKGS = \