Merge branch 'topo' into 'main'

Fix topo sorting of Cargo.toml and reorganise CI jobs a bit

Closes #495

See merge request tpo/core/arti!576

.gitlab-ci.yml

@@ -28,6 +28,32 @@ shellcheck:
     - apk add git bash
     - ./maint/shellcheck_all
 
+maint-checks:
+  stage: check
+  image: debian:stable-slim
+  script:
+    - apt-get update && apt-get install -y python3-toml
+    - ./maint/check_toposort
+    - ./maint/add_warning --check crates/*/src/{lib,main}.rs
+
+rust-checks:
+  # This is too slow (and the cacheing of the "cargo build" too flaky) to be a "check"
+  stage: build
+  image: rust:latest
+  script:
+    - rustup show
+    - rustup component add rustfmt
+    - ./maint/via-cargo-install-in-ci cargo-audit cargo-sort cargo-license
+    - cargo fmt -- --check
+    - ./maint/check_licenses
+    - ./maint/cargo_audit
+    - ./maint/cargo_sort
+  cache:
+    paths:
+      - cargo-audit
+      - cargo-sort
+      - cargo-license
+
 rust-latest:
   stage: build
   image: rust:latest
@@ -35,25 +61,10 @@ rust-latest:
     - rustup show
     - cargo build --locked --verbose --target x86_64-unknown-linux-gnu
     - cargo test --verbose --target x86_64-unknown-linux-gnu
-    - rustup component add clippy rustfmt
+    - rustup component add clippy
     - rustup show
     - cargo clippy --all-features --all-targets -- -D warnings
-    - cargo fmt -- --check
-    - cp cargo-license $CARGO_HOME/bin/ || cargo install cargo-license
-    - ./maint/check_licenses
-    - cp cargo-audit $CARGO_HOME/bin/ || cargo install cargo-audit
-    - ./maint/cargo_audit
-    - cp $CARGO_HOME/bin/cargo-audit .
-    - cp cargo-sort $CARGO_HOME/bin/ || cargo install cargo-sort
-    - ./maint/cargo_sort
-    - cp $CARGO_HOME/bin/cargo-sort .
-    - ./maint/add_warning --check crates/*/src/{lib,main}.rs
     - cargo build --verbose --release -p arti-bench --target x86_64-unknown-linux-gnu
-  cache:
-    paths:
-      - cargo-audit
-      - cargo-sort
-      - cargo-license
   artifacts:
     paths:
       - target/x86_64-unknown-linux-gnu/debug/arti
@@ -98,8 +109,7 @@ coverage:
     - apt-get update && apt-get install -y python3-pip python3-setuptools
     - pip3 install beautifulsoup4 lxml
     - rustup component add llvm-tools-preview
-    - cp grcov $CARGO_HOME/bin/ || cargo install grcov
-    - cp $CARGO_HOME/bin/grcov .
+    - ./maint/via-cargo-install-in-ci grcov
     # Generate report
     - ./maint/with_coverage -f cobertura -o coverage.xml cargo test --verbose --all-features
   cache:

Cargo.toml

@@ -6,41 +6,41 @@
 # Please keep this list topologically sorted by dependency relation, so
 # that every crate appears _before_ any other crate that depends on it.
 members = [
-    "crates/arti",
-    "crates/arti-bench",
+    "crates/tor-basic-utils",
+    "crates/caret",
+    "crates/fs-mistrust",
+    "crates/safelog",
+    "crates/retry-error",
+    "crates/tor-error",
+    "crates/tor-config",
+    "crates/tor-events",
+    "crates/tor-units",
+    "crates/tor-rtcompat",
+    "crates/tor-rtmock",
+    "crates/tor-llcrypto",
+    "crates/tor-protover",
+    "crates/tor-bytes",
+    "crates/tor-socksproto",
+    "crates/tor-checkable",
+    "crates/tor-cert",
+    "crates/tor-linkspec",
+    "crates/tor-cell",
+    "crates/tor-proto",
+    "crates/tor-netdoc",
+    "crates/tor-consdiff",
+    "crates/tor-netdir",
+    "crates/tor-persist",
+    "crates/tor-chanmgr",
+    "crates/tor-guardmgr",
+    "crates/tor-circmgr",
+    "crates/tor-dirclient",
+    "crates/tor-dirmgr",
     "crates/arti-client",
     "crates/arti-config",
     "crates/arti-hyper",
+    "crates/arti",
+    "crates/arti-bench",
     "crates/arti-testing",
-    "crates/caret",
-    "crates/fs-mistrust",
-    "crates/retry-error",
-    "crates/safelog",
-    "crates/tor-basic-utils",
-    "crates/tor-bytes",
-    "crates/tor-cell",
-    "crates/tor-cert",
-    "crates/tor-chanmgr",
-    "crates/tor-checkable",
-    "crates/tor-circmgr",
-    "crates/tor-config",
-    "crates/tor-consdiff",
-    "crates/tor-dirclient",
-    "crates/tor-dirmgr",
-    "crates/tor-error",
-    "crates/tor-events",
-    "crates/tor-guardmgr",
-    "crates/tor-linkspec",
-    "crates/tor-llcrypto",
-    "crates/tor-netdir",
-    "crates/tor-netdoc",
-    "crates/tor-persist",
-    "crates/tor-proto",
-    "crates/tor-protover",
-    "crates/tor-rtcompat",
-    "crates/tor-rtmock",
-    "crates/tor-socksproto",
-    "crates/tor-units",
 ]
 
 resolver = "2"

maint/cargo_sort

@@ -1,7 +1,30 @@
 #!/bin/bash
 #
-# Run "cargo sort" with an appropriate set of flags.
+# Run "cargo sort" to check that the Cargo.tomls are sorted
 
 set -euo pipefail
 
-cargo sort --check --workspace
+# We want to exclude the toplevel Cargo.toml, because that needs to be in
+# topological order.  But cargo sort doesn't support that.
+#  https://github.com/DevinR528/cargo-sort/issues/38
+
+# So instead, we sed its output.  Urgh.
+
+(TERM=dumb cargo sort --check --workspace || test $? = 1) 2>&1 | perl -ne '
+    next if m{^\Qerror: Dependencies for arti are not sorted\E$};
+    $n_arti += !!m{^\QChecking arti...}; # printed for toplevel too
+    next if m{^Checking \S+\Q...\E$};
+    $n_bad++;
+    print STDERR;
+    END {
+        flush STDOUT;
+        eval {
+            die "expected \"Checking arti\" twice, got $n_arti times\n" unless $n_arti==2;
+            die "unexpected output ($n_bad line(s)) from cargo-sort\n" if $n_bad;
+        };
+        if ($@) {
+            print STDERR $@;
+            exit 12;
+        }
+    }
+'

maint/check_toposort

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 #
 # Helper script: makes sure that the crates as listed in Cargo.toml are
 # topologically sorted from lowest-level to highest level.

maint/via-cargo-install-in-ci

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -euo pipefail
+
+for cmd in "$@"; do
+    if cp "$cmd" "$CARGO_HOME"/bin/; then continue; fi
+
+    cargo install "$cmd"
+    cp -v "$CARGO_HOME/bin/$cmd" .
+done