mirror of
https://github.com/microsoft/vcpkg.git
synced 2024-11-28 09:09:23 +08:00
6d3d6490eb
* Remove port version splitting from x-history * Parallelize versions file generator * Use cpu_count()/2 to avoid crashes * Use generatePortVersionsDb.py output to generate baseline * Update scripts/generateBaseline.py Co-authored-by: Adam Johnson <AdamJohnso@gmail.com> * rename generateBaseline function * Update toolsrc/src/vcpkg/commands.porthistory.cpp Co-authored-by: ras0219 <533828+ras0219@users.noreply.github.com> * Remove unused code Co-authored-by: Adam Johnson <AdamJohnso@gmail.com> Co-authored-by: ras0219 <533828+ras0219@users.noreply.github.com>
88 lines
2.9 KiB
Python
88 lines
2.9 KiB
Python
import os
|
|
import sys
|
|
import subprocess
|
|
import json
|
|
import time
|
|
import shutil
|
|
|
|
import multiprocessing
|
|
|
|
from pathlib import Path
|
|
|
|
|
|
MAX_PROCESSES = multiprocessing.cpu_count()
|
|
SCRIPT_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
|
|
PORTS_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, '../ports')
|
|
VERSIONS_DB_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, '../port_versions')
|
|
|
|
|
|
def get_current_git_ref():
|
|
output = subprocess.run(['git', '-C', SCRIPT_DIRECTORY, 'rev-parse', '--verify', 'HEAD'],
|
|
capture_output=True,
|
|
encoding='utf-8')
|
|
if output.returncode == 0:
|
|
return output.stdout.strip()
|
|
print(f"Failed to get git ref:", output.stderr.strip(), file=sys.stderr)
|
|
return None
|
|
|
|
|
|
def generate_port_versions_file(port_name):
|
|
containing_dir = os.path.join(VERSIONS_DB_DIRECTORY, f'{port_name[0]}-')
|
|
os.makedirs(containing_dir, exist_ok=True)
|
|
|
|
output_file_path = os.path.join(containing_dir, f'{port_name}.json')
|
|
if not os.path.exists(output_file_path):
|
|
env = os.environ.copy()
|
|
env['GIT_OPTIONAL_LOCKS'] = '0'
|
|
output = subprocess.run(
|
|
[os.path.join(SCRIPT_DIRECTORY, '../vcpkg.exe'),
|
|
'x-history', port_name, '--x-json', f'--output={output_file_path}'],
|
|
capture_output=True, encoding='utf-8', env=env)
|
|
if output.returncode != 0:
|
|
print(f'x-history {port_name} failed: ',
|
|
output.stdout.strip(), file=sys.stderr)
|
|
|
|
|
|
def generate_port_versions_db(revision):
|
|
start_time = time.time()
|
|
|
|
# Assume each directory in ${VCPKG_ROOT}/ports is a different port
|
|
port_names = [item for item in os.listdir(
|
|
PORTS_DIRECTORY) if os.path.isdir(os.path.join(PORTS_DIRECTORY, item))]
|
|
total_count = len(port_names)
|
|
|
|
concurrency = MAX_PROCESSES / 2
|
|
print(f'Running {concurrency:.0f} parallel processes')
|
|
process_pool = multiprocessing.Pool(MAX_PROCESSES)
|
|
for i, _ in enumerate(process_pool.imap_unordered(generate_port_versions_file, port_names), 1):
|
|
sys.stderr.write(
|
|
f'\rProcessed: {i}/{total_count} ({(i / total_count):.2%})')
|
|
process_pool.close()
|
|
process_pool.join()
|
|
|
|
# Generate timestamp
|
|
rev_file = os.path.join(VERSIONS_DB_DIRECTORY, revision)
|
|
Path(rev_file).touch()
|
|
|
|
elapsed_time = time.time() - start_time
|
|
print(
|
|
f'\nElapsed time: {elapsed_time:.2f} seconds')
|
|
|
|
|
|
def main():
|
|
revision = get_current_git_ref()
|
|
if not revision:
|
|
print('Couldn\'t fetch current Git revision', file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
rev_file = os.path.join(VERSIONS_DB_DIRECTORY, revision)
|
|
if os.path.exists(rev_file):
|
|
print(f'Database files already exist for commit {revision}')
|
|
sys.exit(0)
|
|
|
|
generate_port_versions_db(revision)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|