Skip to content
This repository has been archived by the owner on Dec 18, 2024. It is now read-only.

CI fixes / improvements #478

Merged
merged 2 commits into from
Feb 13, 2023
Merged

CI fixes / improvements #478

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3db5292
Fix clippy::uninlined_format_args lint
argerus Feb 11, 2023
f13b12c
Better error messages in createbom
argerus Feb 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions kuksa_databroker/createbom/bomutil/maplicensefile.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
# SPDX-License-Identifier: Apache-2.0
########################################################################

'''Mapping of liense identifiers of cargo license to the filenames of the actual license texts.'''
"""Mapping of license identifiers of cargo license to the filenames of the actual license texts."""

MAP = {
"Apache-2.0": "Apache-2.0.txt.gz",
Expand All @@ -28,5 +28,5 @@
"BSD-2-Clause": "BSD-2-Clause.txt.gz",
"CC0-1.0": "CC0-1.0.txt.gz",
"WTFPL": "WTFPL.txt.gz",
"Zlib": "Zlib.txt.gz"
"Zlib": "Zlib.txt.gz",
}
173 changes: 113 additions & 60 deletions kuksa_databroker/createbom/createbom.py
100644 → 100755
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#! /usr/bin/env python
#!/usr/bin/env python3
########################################################################
# Copyright (c) 2022 Robert Bosch GmbH
#
Expand All @@ -17,12 +17,12 @@
# SPDX-License-Identifier: Apache-2.0
########################################################################

'''
Thid will generate a list of all dependencies and licenses of a
"""
This script will generate a list of all dependencies and licenses of a
Rust project. It will create a folder called thirdparty in that
project folder containing a list of dependencies and a copy
of each license used in dependencies
'''
"""

import argparse
import sys
Expand All @@ -33,66 +33,119 @@

from subprocess import check_output, CalledProcessError

from bomutil import maplicensefile
from bomutil.maplicensefile import MAP as supported_licenses
from bomutil import quirks

def extract_license_list(all_license_list, component):
'''Extract valid licenses for each dependency. We most of the time
do not care whether it is "AND" or "OR" currently, we currently assume we are
compatible to all "OR" variants, and thus include all'''
component = quirks.apply_quirks(component)
licenses = re.split(r'\s*AND\s*|\s*OR\s*|\(|\)', component["license"])
licenses = list(filter(None, licenses))
print(f"Licenses for {component['name']}: {licenses}")

del component['license_file']

for license_id in licenses:
if license_id not in maplicensefile.MAP:
print(f"BOM creation failed, can not find license text for {license_id} \
used in dependency {component['name']}")
sys.exit(-100)
if maplicensefile.MAP[license_id] not in license_list:
all_license_list.append(maplicensefile.MAP[license_id])

parser = argparse.ArgumentParser()
parser.add_argument("dir", help="Rust project directory")
args = parser.parse_args()

sourcepath = os.path.abspath(args.dir)
targetpath = os.path.join(sourcepath,"thirdparty")

print(f"Generating BOM for project in {sourcepath}")

if os.path.exists(targetpath):
print(f"Folder {targetpath} already exists. Remove it before running this script.")
sys.exit(-2)
class LicenseException(Exception):
pass


try:
cargo_output=check_output(\
["cargo", "license", "--json", "--avoid-build-deps", "--current-dir", sourcepath])
except CalledProcessError as e:
print(f"Error running cargo license: {e}")
sys.exit(-1)
class CargoLicenseException(Exception):
pass


licensedict=json.loads(cargo_output)

license_list = []
for entry in licensedict:
extract_license_list(license_list,entry)

# Exporting
os.mkdir(targetpath)

for licensefile in license_list:
print(f"Copying {licensefile[:-2]}")
with gzip.open("licensestore/"+licensefile, 'rb') as inf:
content = inf.read()
with open(os.path.join(targetpath,licensefile[:-3]),'wb') as outf:
outf.write(content)

print("Writing thirdparty_components.txt")
with open(os.path.join(targetpath,"thirdparty_components.txt"),'w',encoding="utf-8") as jsonout:
json.dump(licensedict,jsonout, indent=4)
def extract_license_ids(crate):
"""Extract valid licenses for each dependency. We most of the time
do not care whether it is "AND" or "OR" currently, we currently assume we are
compatible to all "OR" variants, and thus include all"""
crate = quirks.apply_quirks(crate)
license = crate["license"]

if license:
license_ids = re.split(r"\s*AND\s*|\s*OR\s*|\(|\)", license)
license_ids = list(filter(None, license_ids))
return license_ids
else:
err = f"No license specified for dependency {crate['name']}"
raise LicenseException(err)


def generate_bom(source_path, target_path):
try:
cargo_output = check_output(
[
"cargo",
"license",
"--json",
"--avoid-build-deps",
"--current-dir",
source_path,
]
)
except CalledProcessError as e:
raise CargoLicenseException(f"Error running cargo license: {e}")

crates = json.loads(cargo_output)

license_files = set()
errors = []
for crate in crates:
try:
license_ids = extract_license_ids(crate)
print(f"Licenses for {crate['name']}: {license_ids}")
for license_id in license_ids:
if license_id in supported_licenses:
license_file = supported_licenses[license_id]
license_files.add(license_file)
else:
err = (
f"Could not find license text for {license_id}"
f" used in dependency {crate['name']}"
)
errors.append(err)

except LicenseException as e:
errors.append(e)

if errors:
for error in errors:
print(error)
raise LicenseException("BOM creation failed, unresolved licenses detected")

# Exporting
os.mkdir(target_path)

for license_file in license_files:
print(f"Copying {license_file[:-2]}")
with gzip.open("licensestore/" + license_file, "rb") as inf:
content = inf.read()
with open(os.path.join(target_path, license_file[:-3]), "wb") as outf:
outf.write(content)

print("Writing thirdparty_components.txt")
with open(
os.path.join(target_path, "thirdparty_components.txt"), "w", encoding="utf-8"
) as jsonout:
json.dump(crates, jsonout, indent=4)


def main(args=None):
parser = argparse.ArgumentParser()
parser.add_argument("dir", help="Rust project directory")
args = parser.parse_args(args)

source_path = os.path.abspath(args.dir)
target_path = os.path.join(source_path, "thirdparty")

if os.path.exists(target_path):
print(
f"Folder {target_path} already exists. Remove it before running this script."
)
return -2

print(f"Generating BOM for project in {source_path}")
try:
generate_bom(source_path, target_path)
except LicenseException as e:
print(f"Error: {e}")
return -100
except CargoLicenseException as e:
print(f"Error: {e}")
return -1


if __name__ == "__main__":
import sys

sys.exit(main(sys.argv[1:]))
52 changes: 21 additions & 31 deletions kuksa_databroker/databroker-cli/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ async fn connect(
}
Err(err) => {
set_disconnected_prompt(interface);
writeln!(interface, "{}", err).unwrap();
writeln!(interface, "{err}").unwrap();
None
}
}
Expand Down Expand Up @@ -132,7 +132,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {

let addr = std::env::var("KUKSA_DATA_BROKER_ADDR").unwrap_or_else(|_| "127.0.0.1".to_owned());
let port = std::env::var("KUKSA_DATA_BROKER_PORT").unwrap_or_else(|_| "55555".to_owned());
let mut uri = match addr_to_uri(format!("{}:{}", addr, port)) {
let mut uri = match addr_to_uri(format!("{addr}:{port}")) {
Some(uri) => uri,
None => return Err(Box::new(ParseError {}).into()),
};
Expand All @@ -154,10 +154,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let (cmd, args) = split_first_word(&line);
match cmd {
"help" => {
println!("{} commands:", APP_NAME);
println!("{APP_NAME} commands:");
println!();
for &(cmd, help) in CLI_COMMANDS {
println!(" {:15} - {}", cmd, help);
println!(" {cmd:15} - {help}");
}
println!();
}
Expand Down Expand Up @@ -270,15 +270,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
) {
Some(error) => {
println!(
"-> Error setting id {}: {:?}",
id, error
"-> Error setting id {id}: {error:?}"
)
}
None => {
println!(
"-> Error setting id {}",
id
)
println!("-> Error setting id {id}")
}
}
}
Expand Down Expand Up @@ -367,15 +363,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
) {
Some(error) => {
println!(
"-> Error setting id {}: {:?}",
id, error
"-> Error setting id {id}: {error:?}"
)
}
None => {
println!(
"-> Error setting id {}",
id
)
println!("-> Error setting id {id}")
}
}
}
Expand Down Expand Up @@ -417,8 +409,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
tonic::Request::new(proto::v1::SubscribeRequest { query });
match client.subscribe(args).await {
Ok(response) => {
let sub_id =
format!("subscription{}", subscription_nbr);
let sub_id = format!("subscription{subscription_nbr}");
subscription_nbr += 1;
tokio::spawn(async move {
let mut stream = response.into_inner();
Expand All @@ -441,8 +432,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
}
writeln!(
iface,
"-> {}:\n{}",
sub_id, output
"-> {sub_id}:\n{output}"
)
.unwrap();
}
Expand Down Expand Up @@ -531,7 +521,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
break;
}
ReadResult::Signal(sig) => {
println!("received signal: {:?}", sig);
println!("received signal: {sig:?}");
}
}
}
Expand Down Expand Up @@ -778,9 +768,9 @@ where
let real_delimiter = ", ";
let mut delimiter = "";
for value in array {
write!(f, "{}", delimiter)?;
write!(f, "{delimiter}")?;
delimiter = real_delimiter;
write!(f, "{}", value)?;
write!(f, "{value}")?;
}
f.write_str("]")
}
Expand All @@ -790,32 +780,32 @@ impl fmt::Display for DisplayDatapoint {
match &self.0.value {
Some(value) => match value {
proto::v1::datapoint::Value::BoolValue(value) => {
f.write_fmt(format_args!("{}", value))
f.write_fmt(format_args!("{value}"))
}
proto::v1::datapoint::Value::FailureValue(failure) => f.write_fmt(format_args!(
"( {:?} )",
proto::v1::datapoint::Failure::from_i32(*failure).unwrap()
)),
proto::v1::datapoint::Value::Int32Value(value) => {
f.write_fmt(format_args!("{}", value))
f.write_fmt(format_args!("{value}"))
}
proto::v1::datapoint::Value::Int64Value(value) => {
f.write_fmt(format_args!("{}", value))
f.write_fmt(format_args!("{value}"))
}
proto::v1::datapoint::Value::Uint32Value(value) => {
f.write_fmt(format_args!("{}", value))
f.write_fmt(format_args!("{value}"))
}
proto::v1::datapoint::Value::Uint64Value(value) => {
f.write_fmt(format_args!("{}", value))
f.write_fmt(format_args!("{value}"))
}
proto::v1::datapoint::Value::FloatValue(value) => {
f.write_fmt(format_args!("{:.2}", value))
f.write_fmt(format_args!("{value:.2}"))
}
proto::v1::datapoint::Value::DoubleValue(value) => {
f.write_fmt(format_args!("{}", value))
f.write_fmt(format_args!("{value}"))
}
proto::v1::datapoint::Value::StringValue(value) => {
f.write_fmt(format_args!("'{}'", value))
f.write_fmt(format_args!("'{value}'"))
}
proto::v1::datapoint::Value::StringArray(array) => display_array(f, &array.values),
proto::v1::datapoint::Value::BoolArray(array) => display_array(f, &array.values),
Expand Down
Loading