build-pkg: add second pass

The second pass is done after the first one. During the second pass,
an item is built in tree of files from all other items. It checks that:

  * packages can be built in any order satisfying dependencies;
  * a package can be rebuilt after its dependee.

For both cases, not only build status is checked but also two sets of
files are compared. Currently content of files is not checked, only
their existance in both passes.

See #1111

tools/build-pkg.lua

@@ -548,6 +548,57 @@ local function removeEmptyDirs(item)
     end
 end
 
+local function prepareTree(pass, item, item2deps, prev_files, item2index)
+    if pass == 'first' then
+        gitCheckout(
+            itemToBranch(item, pass),
+            item2deps[item],
+            item2index,
+            pass
+        )
+    elseif pass == 'second' then
+        -- Build item second time to check if it builds correctly if
+        -- its followers and unrelated packages have been built.
+        gitCheckout(
+            itemToBranch(item, 'second'),
+            {GIT_ALL_PSEUDOITEM},
+            item2index,
+            'first'
+        )
+        -- Remove files of item from previous build.
+        for _, file in ipairs(prev_files) do
+            os.remove(file)
+        end
+        removeEmptyDirs()
+        gitAdd()
+        gitCommit(("Remove %s to rebuild it"):format(item, pass))
+    else
+        error("Unknown pass: " .. pass)
+    end
+end
+
+local function comparePasses(item, new_files, prev_file2item, prev_files)
+    local files_set = {}
+    for _, file in ipairs(new_files) do
+        if not prev_file2item[file] then
+            log('Item %s installs a file on second pass only: %s',
+                item, file)
+        elseif prev_file2item[file] ~= item then
+            log('File %s was installed by %s on first pass ' ..
+                'and by %s - on the second pass',
+                file, prev_file2item[file], item)
+        end
+        files_set[file] = true
+    end
+    for _, file in ipairs(prev_files) do
+        if not files_set[file] then
+            log('Item %s installs a file on first pass only: %s',
+                item, file)
+        end
+    end
+    -- TODO compare contents of files (nm for binaries)
+end
+
 local function isBuilt(item, files)
     local target, pkg = parseItem(item)
     local INSTALLED = 'usr/%s/installed/%s'
@@ -561,10 +612,9 @@ local function isBuilt(item, files)
 end
 
 -- builds package, returns list of new files
-local function buildItem(item, item2deps, file2item, item2index, pass)
-    gitCheckout(
-        itemToBranch(item, pass), item2deps[item], item2index, pass
-    )
+-- prev_files is passed only to second pass.
+local function buildItem(item, item2deps, file2item, item2index, pass, prev_files)
+    prepareTree(pass, item, item2deps, prev_files, item2index)
     local target, pkg = parseItem(item)
     local cmd = '%s %s MXE_TARGETS=%s --jobs=1'
     os.execute(cmd:format(tool 'make', pkg, target))
@@ -573,9 +623,13 @@ local function buildItem(item, item2deps, file2item, item2index, pass)
     if #new_files + #changed_files > 0 then
         gitCommit(("Build %s, pass %s"):format(item, pass))
     end
-    for _, file in ipairs(new_files) do
-        checkFile(file, item)
-        file2item[file] = item
+    if pass == 'first' then
+        for _, file in ipairs(new_files) do
+            checkFile(file, item)
+            file2item[file] = item
+        end
+    elseif isBuilt(item, new_files) then
+        comparePasses(item, new_files, file2item, prev_files)
     end
     for _, file in ipairs(changed_files) do
         checkFile(file, item)
@@ -776,7 +830,8 @@ local function isEmpty(files)
 end
 
 -- build all packages, save filelist to list file
-local function buildPackages(items, item2deps, pass)
+-- prev_files is passed only to second pass.
+local function buildPackages(items, item2deps, pass, prev_item2files)
     local broken = {}
     local unbroken = {}
     local file2item = {}
@@ -789,12 +844,22 @@ local function buildPackages(items, item2deps, pass)
         end
         return false
     end
+    if pass == 'second' then
+        assert(prev_item2files)
+        -- fill file2item with data from prev_item2files
+        for item, files in pairs(prev_item2files) do
+            for _, file in ipairs(files) do
+                file2item[file] = item
+            end
+        end
+    end
     local item2index = makeItem2Index(items)
     local progress_printer = progressPrinter(items)
     for i, item in ipairs(items) do
         if not brokenDep(item) then
+            local prev_files = prev_item2files and prev_item2files[item]
             local files = buildItem(
-                item, item2deps, file2item, item2index, pass
+                item, item2deps, file2item, item2index, pass, prev_files
             )
             findForeignInstalls(item, files)
             if isBuilt(item, files) then
@@ -959,6 +1024,10 @@ local function main()
         makeMxeRequirementsPackage('jessie')
     end
     makeMxeSourcePackage()
+    -- second pass
+    buildPackages(
+        build_list, item2deps, 'second', item2files
+    )
     if #unbroken < #build_list then
         local code = 1
         local close = true