-
Notifications
You must be signed in to change notification settings - Fork 280
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
build: refactor JSON info files to
profiles.json
JSON info files contain machine readable information of built profiles and resulting images. These files were added in commit 881ed09 ("build: create JSON files containing image info"). They are useful for firmware wizards and script checking for reproducibility. Currently all JSON files are stored next to the built images, resulting in up to 168 individual files for the ath79/generic target. This patch refactors the JSON creation to store individual per image (not per profile) files in $(BUILD_DIR)/json_info_files and create an single overview file called `profiles.json` in the target directory. Storing per image files and not per profile solves the problem of parallel file writes. If a profiles sysupgrade and factory image are finished at the same time both processes would write to the same JSON file, resulting in randomly broken outputs. Some target like x86/64 do not use the image code yet, resulting in missing JSON files. If no JSON info files were created, no `profiles.json` files is created as it would be empty anyway. As before, this creation is enabled by default only if `BUILDBOT` is set. Tested via buildroot & ImageBuilder on ath79/generic, imx6 and x86/64. Signed-off-by: Paul Spooren <[email protected]> [json_info_files dir handling in Make, if case refactoring] Signed-off-by: Petr Štetiar <[email protected]> (backported from commit 07449f6) Signed-off-by: Adrian Schmutzler <[email protected]>
- Loading branch information
Showing
6 changed files
with
124 additions
and
54 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,42 +1,50 @@ | ||
| #!/usr/bin/env python3 | ||
|
|
||
| import json | ||
| import os | ||
| from os import getenv | ||
| from pathlib import Path | ||
| from sys import argv | ||
| import hashlib | ||
| import json | ||
|
|
||
| if len(argv) != 2: | ||
| print("ERROR: JSON info script requires output arg") | ||
| exit(1) | ||
|
|
||
| def e(variable, default=None): | ||
| return os.environ.get(variable, default) | ||
|
|
||
|
|
||
| json_path = "{}{}{}.json".format(e("BIN_DIR"), os.sep, e("IMAGE_PREFIX")) | ||
| json_path = Path(argv[1]) | ||
| bin_dir = Path(getenv("BIN_DIR")) | ||
| image_file = bin_dir / getenv("IMAGE_NAME") | ||
|
|
||
| with open(os.path.join(e("BIN_DIR"), e("IMAGE_NAME")), "rb") as image_file: | ||
| image_hash = hashlib.sha256(image_file.read()).hexdigest() | ||
| if not image_file.is_file(): | ||
| print("Skip JSON creation for non existing image ", image_file) | ||
| exit(0) | ||
|
|
||
|
|
||
| def get_titles(): | ||
| return [{"title": e("DEVICE_TITLE")}] | ||
|
|
||
|
|
||
| if not os.path.exists(json_path): | ||
| device_info = { | ||
| "id": e("DEVICE_ID"), | ||
| "image_prefix": e("IMAGE_PREFIX"), | ||
| "images": [], | ||
| "metadata_version": 1, | ||
| "supported_devices": e("SUPPORTED_DEVICES").split(), | ||
| "target": "{}/{}".format(e("TARGET"), e("SUBTARGET", "generic")), | ||
| "titles": get_titles(), | ||
| "version_commit": e("VERSION_CODE"), | ||
| "version_number": e("VERSION_NUMBER"), | ||
| } | ||
| else: | ||
| with open(json_path, "r") as json_file: | ||
| device_info = json.load(json_file) | ||
|
|
||
| image_info = {"type": e("IMAGE_TYPE"), "name": e("IMAGE_NAME"), "sha256": image_hash} | ||
| device_info["images"].append(image_info) | ||
|
|
||
| with open(json_path, "w") as json_file: | ||
| json.dump(device_info, json_file, sort_keys=True, indent=" ") | ||
| return [{"title": getenv("DEVICE_TITLE")}] | ||
|
|
||
|
|
||
| device_id = getenv("DEVICE_ID") | ||
| image_hash = hashlib.sha256(image_file.read_bytes()).hexdigest() | ||
|
|
||
| image_info = { | ||
| "metadata_version": 1, | ||
| "target": "{}/{}".format(getenv("TARGET"), getenv("SUBTARGET")), | ||
| "version_code": getenv("VERSION_CODE"), | ||
| "version_number": getenv("VERSION_NUMBER"), | ||
| "profiles": { | ||
| device_id: { | ||
| "image_prefix": getenv("IMAGE_PREFIX"), | ||
| "images": [ | ||
| { | ||
| "type": getenv("IMAGE_TYPE"), | ||
| "name": getenv("IMAGE_NAME"), | ||
| "sha256": image_hash, | ||
| } | ||
| ], | ||
| "supported_devices": getenv("SUPPORTED_DEVICES").split(), | ||
| "titles": get_titles(), | ||
| } | ||
| }, | ||
| } | ||
|
|
||
| json_path.write_text(json.dumps(image_info, separators=(",", ":"))) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| #!/usr/bin/env python3 | ||
|
|
||
| import json | ||
| from pathlib import Path | ||
| from os import getenv | ||
| from sys import argv | ||
|
|
||
| if len(argv) != 2: | ||
| print("JSON info files script requires ouput file as argument") | ||
| exit(1) | ||
|
|
||
| output_path = Path(argv[1]) | ||
|
|
||
| assert getenv("WORK_DIR"), "$WORK_DIR required" | ||
|
|
||
| work_dir = Path(getenv("WORK_DIR")) | ||
|
|
||
| assert work_dir.is_dir(), "$WORK_DIR not a directory" | ||
|
|
||
| output = {} | ||
|
|
||
| for json_file in work_dir.glob("*.json"): | ||
| image_info = json.loads(json_file.read_text()) | ||
| if not output: | ||
| output.update(image_info) | ||
| else: | ||
| # get first (and only) profile in json file | ||
| device_id = next(iter(image_info["profiles"].keys())) | ||
| if device_id not in output["profiles"]: | ||
| output["profiles"].update(image_info["profiles"]) | ||
| else: | ||
| output["profiles"][device_id]["images"].append( | ||
| image_info["profiles"][device_id]["images"][0] | ||
| ) | ||
|
|
||
| if output: | ||
| output_path.write_text(json.dumps(output, sort_keys=True, separators=(",", ":"))) | ||
| else: | ||
| print("JSON info file script could not find any JSON files for target") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters