From 31d151aa3021b0d4606aff08103c9a671765b09e Mon Sep 17 00:00:00 2001 From: Christoffer Lerno Date: Sun, 14 Aug 2022 23:42:18 +0200 Subject: [PATCH] Added MSVC download script to prebuilts. Implicitly use msvc_sdk if available. Bump version to 0.3.17 --- .github/workflows/main.yml | 6 + ...stall_win_reqs.bat => install_win_reqs.bat | 34 +-- msvc_build_libraries.py | 233 ++++++++++++++++++ src/compiler/compiler_internal.h | 1 + src/compiler/linker.c | 28 +++ src/compiler/windows_support.c | 7 +- src/utils/file_utils.c | 29 +++ src/utils/lib.h | 1 + src/version.h | 2 +- 9 files changed, 322 insertions(+), 19 deletions(-) rename resources/install_win_reqs.bat => install_win_reqs.bat (97%) create mode 100755 msvc_build_libraries.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5e751a71f..424c57829 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -188,6 +188,7 @@ jobs: run: | mkdir linux cp -r lib/ linux + cp msvc_build_libraries.py linux cp build/c3c linux tar czf c3-linux-${{matrix.build_type}}.tar.gz linux @@ -241,6 +242,7 @@ jobs: run: | mkdir macos cp -r lib/ macos + cp msvc_build_libraries.py linux cp build/c3c macos zip -r c3-macos-${{matrix.build_type}}.zip macos @@ -284,6 +286,10 @@ jobs: - uses: actions/download-artifact@v3 - run: cp -r lib/ c3-windows-Release - run: cp -r lib/ c3-windows-Debug + - run: cp msvc_build_libraries.py c3-windows-Release + - run: cp msvc_build_libraries.py c3-windows-Debug + - run: cp install_win_reqs.bat c3-windows-Release + - run: cp install_win_reqs.bat c3-windows-Debug - run: zip -r c3-windows-Release.zip c3-windows-Release - run: zip -r c3-windows-Debug.zip c3-windows-Debug diff --git a/resources/install_win_reqs.bat b/install_win_reqs.bat similarity index 97% rename from resources/install_win_reqs.bat rename to install_win_reqs.bat index b6f5a7155..acbc4e273 100644 --- a/resources/install_win_reqs.bat +++ b/install_win_reqs.bat @@ -1,17 +1,17 @@ -@echo off - -set DOWNLOAD_URL=https://aka.ms/vs/17/release - -mkdir tmp 2> NUL - -if not exist "tmp\vs_buildtools.exe" ( - bitsadmin /transfer /download /priority foreground %DOWNLOAD_URL%/vs_buildtools.exe %CD%\tmp\vs_buildtools.exe -) - -echo Preparing Build Tools, please wait... -tmp\vs_BuildTools.exe --quiet --wait --layout tmp\ --add Microsoft.VisualStudio.Component.Windows10SDK.19041 - -echo Installing Build Tools, please wait... -tmp\vs_BuildTools.exe --quiet --wait --noweb --add Microsoft.VisualStudio.Component.Windows10SDK.19041 - -REM rmdir tmp /s /q +@echo off + +set DOWNLOAD_URL=https://aka.ms/vs/17/release + +mkdir tmp 2> NUL + +if not exist "tmp\vs_buildtools.exe" ( + bitsadmin /transfer /download /priority foreground %DOWNLOAD_URL%/vs_buildtools.exe %CD%\tmp\vs_buildtools.exe +) + +echo Preparing Build Tools, please wait... +tmp\vs_BuildTools.exe --quiet --wait --layout tmp\ --add Microsoft.VisualStudio.Component.Windows10SDK.19041 + +echo Installing Build Tools, please wait... +tmp\vs_BuildTools.exe --quiet --wait --noweb --add Microsoft.VisualStudio.Component.Windows10SDK.19041 + +REM rmdir tmp /s /q diff --git a/msvc_build_libraries.py b/msvc_build_libraries.py new file mode 100755 index 000000000..7c64c6a23 --- /dev/null +++ b/msvc_build_libraries.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +import platform +import io +import os +import sys +import json +import shutil +import hashlib +import zipfile +import tempfile +import argparse +import subprocess +import urllib.request +import re +from pathlib import Path + +OUTPUT = Path("msvc_temp") # output folder +SDK_OUTPUT = Path("msvc_sdk") + +MANIFEST_URL = "https://aka.ms/vs/17/release/channel" + +def download(url): + with urllib.request.urlopen(url) as res: + return res.read() + +def download_progress(url, check, name, f): + data = io.BytesIO() + with urllib.request.urlopen(url) as res: + total = int(res.headers["Content-Length"]) + size = 0 + while True: + block = res.read(1<<20) + if not block: + break + f.write(block) + data.write(block) + size += len(block) + perc = size * 100 // total + print(f"\r{name} ... {perc}%", end="") + print() + data = data.getvalue() + digest = hashlib.sha256(data).hexdigest() + if check.lower() != digest: + exit(f"Hash mismatch for f{pkg}") + return data + +# super crappy msi format parser just to find required .cab files +def get_msi_cabs(msi): + index = 0 + while True: + index = msi.find(b".cab", index+4) + if index < 0: + return + yield msi[index-32:index+4].decode("ascii") + +def first(items, cond): + return next(item for item in items if cond(item)) + + +### parse command-line arguments + +ap = argparse.ArgumentParser() +ap.add_argument("--show-versions", const=True, action="store_const", help="Show available MSVC and Windows SDK versions") +ap.add_argument("--accept-license", const=True, action="store_const", help="Automatically accept license") +ap.add_argument("--msvc-version", help="Get specific MSVC version") +ap.add_argument("--sdk-version", help="Get specific Windows SDK version") +args = ap.parse_args() + + +### get main manifest + +manifest = json.loads(download(MANIFEST_URL)) + + +### download VS manifest + +vs = first(manifest["channelItems"], lambda x: x["id"] == "Microsoft.VisualStudio.Manifests.VisualStudio") +payload = vs["payloads"][0]["url"] + +vsmanifest = json.loads(download(payload)) + + +### find MSVC & WinSDK versions + +packages = {} +for p in vsmanifest["packages"]: + packages.setdefault(p["id"].lower(), []).append(p) + +msvc = {} +sdk = {} + +for pid,p in packages.items(): + if pid.startswith("Microsoft.VisualStudio.Component.VC.".lower()) and pid.endswith(".x86.x64".lower()): + pver = ".".join(pid.split(".")[4:6]) + if pver[0].isnumeric(): + msvc[pver] = pid + elif pid.startswith("Microsoft.VisualStudio.Component.Windows10SDK.".lower()) or \ + pid.startswith("Microsoft.VisualStudio.Component.Windows11SDK.".lower()): + pver = pid.split(".")[-1] + if pver.isnumeric(): + sdk[pver] = pid + +if args.show_versions: + print("MSVC versions:", " ".join(sorted(msvc.keys()))) + print("Windows SDK versions:", " ".join(sorted(sdk.keys()))) + exit(0) + +msvc_ver = args.msvc_version or max(sorted(msvc.keys())) +sdk_ver = args.sdk_version or max(sorted(sdk.keys())) + +if msvc_ver in msvc: + msvc_pid = msvc[msvc_ver] + msvc_ver = ".".join(msvc_pid.split(".")[4:-2]) +else: + exit(f"Unknown MSVC version: f{args.msvc_version}") + +if sdk_ver in sdk: + sdk_pid = sdk[sdk_ver] +else: + exit(f"Unknown Windows SDK version: f{args.sdk_version}") + +print(f"Downloading MSVC v{msvc_ver} and Windows SDK v{sdk_ver}") + + +### agree to license + +tools = first(manifest["channelItems"], lambda x: x["id"] == "Microsoft.VisualStudio.Product.BuildTools") +resource = first(tools["localizedResources"], lambda x: x["language"] == "en-us") +license = resource["license"] + +if not args.accept_license: + accept = input(f"Do you accept Visual Studio license at {license}, and also confirm that you have a valid license Visual Studio license allowing you to download the VS Build Tools [Y/N] ?") + if not accept or accept[0].lower() != "y": + exit(0) + +shutil.rmtree(OUTPUT, ignore_errors = True) +shutil.rmtree(SDK_OUTPUT, ignore_errors = True) +OUTPUT.mkdir() +total_download = 0 + +### download Windows SDK + +archs = [ + #"arm", + #"arm64", + "x64", + #"x86" +] + +msvc_packages = [ + f"microsoft.vc.{msvc_ver}.asan.headers.base", +] + +for arch in archs: + msvc_packages.append(f"microsoft.vc.{msvc_ver}.crt.{arch}.desktop.base") + msvc_packages.append(f"microsoft.vc.{msvc_ver}.crt.{arch}.store.base") + msvc_packages.append(f"microsoft.vc.{msvc_ver}.asan.{arch}.base") + +for pkg in msvc_packages: + p = first(packages[pkg], lambda p: p.get("language") in (None, "en-US")) + for payload in p["payloads"]: + with tempfile.TemporaryFile() as f: + data = download_progress(payload["url"], payload["sha256"], pkg, f) + total_download += len(data) + with zipfile.ZipFile(f) as z: + for name in z.namelist(): + if name.startswith("Contents/"): + out = OUTPUT / Path(name).relative_to("Contents") + out.parent.mkdir(parents=True, exist_ok=True) + out.write_bytes(z.read(name)) + +sdk_packages = [ + # Windows SDK libs + f"Windows SDK for Windows Store Apps Libs-x86_en-us.msi", + f"Windows SDK Desktop Libs x64-x86_en-us.msi", + # CRT headers & libs + f"Universal CRT Headers Libraries and Sources-x86_en-us.msi", +] + +with tempfile.TemporaryDirectory() as d: + dst = Path(d) + + sdk_pkg = packages[sdk_pid][0] + sdk_pkg = packages[first(sdk_pkg["dependencies"], lambda x: True).lower()][0] + + msi = [] + cabs = [] + + # download msi files + for pkg in sdk_packages: + payload = first(sdk_pkg["payloads"], lambda p: p["fileName"] == f"Installers\\{pkg}") + msi.append(dst / pkg) + with open(dst / pkg, "wb") as f: + data = download_progress(payload["url"], payload["sha256"], pkg, f) + total_download += len(data) + cabs += list(get_msi_cabs(data)) + + # download .cab files + for pkg in cabs: + payload = first(sdk_pkg["payloads"], lambda p: p["fileName"] == f"Installers\\{pkg}") + with open(dst / pkg, "wb") as f: + download_progress(payload["url"], payload["sha256"], pkg, f) + + print("Unpacking msi files...") + + # run msi installers + for m in msi: + if (platform.system() == "Windows"): + subprocess.check_call(["msiexec.exe", "/a", m, "/quiet", "/qn", f"TARGETDIR={OUTPUT.resolve()}"]) + else: + subprocess.check_call(["msiextract", m, '-C', OUTPUT.resolve()]) + + +### versions + +ucrt = list((OUTPUT / "Program Files/Windows Kits/").glob("*/Lib/*/ucrt"))[0] +um = list((OUTPUT / "Program Files/Windows Kits/").glob("*/Lib/*/um"))[0] +lib = list((OUTPUT / "VC/Tools/MSVC/").glob("*/lib"))[0] + +SDK_OUTPUT.mkdir(exist_ok=True) + + + +for arch in archs: + out_dir = SDK_OUTPUT / arch + shutil.copytree(ucrt / arch, out_dir, dirs_exist_ok=True) + shutil.copytree(um / arch, out_dir, dirs_exist_ok=True) + shutil.copytree(lib / arch, out_dir, dirs_exist_ok=True) + +### cleanup + +shutil.rmtree(OUTPUT, ignore_errors=True) + diff --git a/src/compiler/compiler_internal.h b/src/compiler/compiler_internal.h index c6d7f87c4..765b97d71 100644 --- a/src/compiler/compiler_internal.h +++ b/src/compiler/compiler_internal.h @@ -2057,6 +2057,7 @@ bool os_is_apple(OsType os_type); const char *macos_sysroot(void); MacSDK *macos_sysroot_sdk_information(const char *sdk_path); WindowsSDK *windows_get_sdk(void); +const char *windows_cross_compile_library(void); INLINE bool visible_external(Visibility visibility); void c_abi_func_create(FunctionPrototype *proto); diff --git a/src/compiler/linker.c b/src/compiler/linker.c index b45dbc8eb..24638d4a7 100644 --- a/src/compiler/linker.c +++ b/src/compiler/linker.c @@ -84,6 +84,34 @@ static void linker_setup_windows(const char ***args_ref, LinkerType linker_type) default: UNREACHABLE } + if (!active_target.win.sdk) + { + const char *path = windows_cross_compile_library(); + if (path) + { + switch (platform_target.arch) + { + case ARCH_TYPE_ARM: + scratch_buffer_append("/arm"); + break; + case ARCH_TYPE_AARCH64: + scratch_buffer_append("/arm64"); + break; + case ARCH_TYPE_X86_64: + scratch_buffer_append("/x64"); + break; + case ARCH_TYPE_X86: + scratch_buffer_append("/x86"); + break; + default: + UNREACHABLE + } + if (file_exists(scratch_buffer_to_string())) + { + active_target.win.sdk = scratch_buffer_copy(); + } + } + } if (active_target.win.sdk) { add_arg(str_printf("/LIBPATH:%s", active_target.win.sdk)); diff --git a/src/compiler/windows_support.c b/src/compiler/windows_support.c index 80d9998b2..871815011 100644 --- a/src/compiler/windows_support.c +++ b/src/compiler/windows_support.c @@ -37,4 +37,9 @@ WindowsSDK *windows_get_sdk(void) return NULL; } -#endif \ No newline at end of file +#endif + +const char *windows_cross_compile_library(void) +{ + return find_rel_exe_dir("msvc_sdk"); +} \ No newline at end of file diff --git a/src/utils/file_utils.c b/src/utils/file_utils.c index 44361159f..55a2c91ca 100644 --- a/src/utils/file_utils.c +++ b/src/utils/file_utils.c @@ -255,6 +255,35 @@ static inline const char *lib_find(const char *exe_path, const char *rel_path) return lib_path; } +const char *find_rel_exe_dir(const char *dir) +{ + char *path = find_executable_path(); + DEBUG_LOG("Detected executable path at %s", path); + size_t strlen_path = strlen(path); + // Remove any last path slash + if (strlen_path > 1 && (path[strlen_path - 1] == '/' || path[strlen_path - 1] == '\\')) + { + path[strlen_path - 1] = '\0'; + } + struct stat info; + const char *attempts[5] = { "/../", "/lib/", "/../lib/", "/", "/../../lib/" }; + + for (size_t i = 0; i < 5; i++) + { + scratch_buffer_clear(); + scratch_buffer_printf("%s%s%s", path, attempts[i], dir); + DEBUG_LOG("Checking %s", scratch_buffer_to_string()); + int err = stat(scratch_buffer_to_string(), &info); + + // Not a dir or had error? + if (err || !S_ISDIR(info.st_mode)) continue; + return scratch_buffer_to_string(); + } + return NULL; + + +} + const char *find_lib_dir(void) { diff --git a/src/utils/lib.h b/src/utils/lib.h index 378662b12..cf8add180 100644 --- a/src/utils/lib.h +++ b/src/utils/lib.h @@ -65,6 +65,7 @@ bool dir_change(const char *path); bool file_namesplit(const char *path, char** filename_ptr, char** directory_ptr); const char* file_expand_path(const char* path); const char* find_lib_dir(void); +const char *find_rel_exe_dir(const char *dir); bool file_delete_all_files_in_dir_with_suffix(const char *dir, const char *suffix); bool file_is_dir(const char *file); bool file_exists(const char *path); diff --git a/src/version.h b/src/version.h index 4d2d62fd2..a95a72579 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define COMPILER_VERSION "0.3.16" \ No newline at end of file +#define COMPILER_VERSION "0.3.17" \ No newline at end of file