[vcpkg] Added python script to generate all packages file list, added to azur… (#12177)

* Added python script to generate all packages file list, added to azure pipeline to build at the end of the run

* fixed an issue that causes some leading slashes were being removed

* Header Database now includes hpp files for C++ header files

* Changed Header Database to include all files under \include\ folder

* Apply suggestions from code review, changing condition from eq to ne

Co-authored-by: Billy O'Neal <bion@microsoft.com>

* Update last condition from eq to ne

Co-authored-by: Billy O'Neal <bion@microsoft.com>

scripts/azure-pipelines/linux/azure-pipelines.yml

@@ -48,3 +48,15 @@ jobs:
       PathtoPublish: '$(System.ArtifactsDirectory)/failure-logs'
       ArtifactName: 'x64-linux port build failure logs'
     condition: failed()
+  - bash: |
+      python3 scripts/file_script.py /mnt/vcpkg-ci/installed/vcpkg/info/
+    displayName: 'Build a file list for all packages'
+    condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+
+  - task: PublishBuildArtifacts@1
+    displayName: 'Upload file lists for all packages'
+    condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+
+    inputs:
+      PathtoPublish: scripts/list_files
+      ArtifactName: "x64-linux package file lists"

scripts/azure-pipelines/osx/azure-pipelines.yml

@@ -52,3 +52,15 @@ jobs:
       PathtoPublish: '$(System.ArtifactsDirectory)/failure-logs'
       ArtifactName: 'x64-osx port build failure logs'
     condition: failed()
+  - bash: |
+      python3 scripts/file_script.py /Users/vagrant/Data/installed/vcpkg/info/
+    displayName: 'Build a file list for all packages'
+    condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+
+  - task: PublishBuildArtifacts@1
+    displayName: 'Upload file lists for all packages'
+    condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+
+    inputs:
+      PathtoPublish: scripts/list_files
+      ArtifactName: "x64-osx package file lists"

scripts/azure-pipelines/windows/azure-pipelines.yml

@@ -61,3 +61,19 @@ jobs:
       PathtoPublish: '$(System.ArtifactsDirectory)\failure-logs'
       ArtifactName: '${{ parameters.triplet }} port build failure logs'
     condition: failed()
+  - task: PowerShell@2
+    displayName: "Generating all packages files"
+    condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+
+    inputs:
+      targetType: inline
+      script: |
+        $env:VCPKG_DOWNLOADS = "D:\downloads"
+        ./vcpkg.exe fetch python3
+        & $(.\vcpkg fetch python3) .\scripts\file_script.py D:\installed\vcpkg\info\
+  - task: PublishBuildArtifacts@1
+    displayName: 'Upload file lists for all packages'
+    condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+    inputs:
+      PathtoPublish: scripts/list_files
+      ArtifactName: "${{ parameters.triplet }} package file lists"

scripts/file_script.py

@@ -0,0 +1,39 @@
+import os
+import os.path
+import sys
+
+
+keyword = "include/"
+
+def getFiles(path):
+    files = os.listdir(path)
+    return list(filter(lambda x: x[0] != '.', files))
+
+def gen_all_file_strings(path, files, headers, output):
+    for file in files:
+        package = file[:file.find("_")]
+        f = open(path + file)
+        for line in f:
+            idx = line.strip().find(keyword)
+            if idx >= 0 and line.strip()[-1] != "/":
+                headers.write(package + ":" + line[idx + len(keyword):])
+                output.write(package + ":" + line[idx-1:])
+            elif line.strip()[-1] != "/":
+                output.write(package + ":" + line[line.find("/"):])
+        f.close()
+
+def main(path):
+    try:
+        os.mkdir("scripts/list_files")
+    except FileExistsError:
+        print("Path already exists, continuing...")
+
+    headers = open("scripts/list_files/VCPKGHeadersDatabase.txt", mode='w')
+    output = open("scripts/list_files/VCPKGDatabase.txt", mode='w')
+    gen_all_file_strings(path, getFiles(path), headers, output)
+    headers.close()
+    output.close()
+
+if __name__ == "__main__":
+    main(sys.argv[1])
+

scripts/vcpkgTools.xml

@@ -1,5 +1,12 @@
 <?xml version="1.0"?>
 <tools version="2">
+    <tool name="python3" os="windows">
+        <version>3.8.3</version>
+        <exeRelativePath>python.exe</exeRelativePath>
+        <url>https://www.python.org/ftp/python/3.8.3/python-3.8.3-embed-win32.zip</url>
+        <sha512>8c9078f55b1b5d694e0e809eee6ccf8a6e15810dd4649e8ae1209bff30e102d49546ce970a5d519349ca7759d93146f459c316dc440737171f018600255dcd0a</sha512>
+        <archiveName>python-3.8.3-embed-win32.zip</archiveName>
+    </tool>
     <tool name="cmake" os="windows">
         <version>3.17.2</version>
         <exeRelativePath>cmake-3.17.2-win32-x86\bin\cmake.exe</exeRelativePath>