switch from awk to python

This commit is contained in:
trinity-1686a 2022-08-24 18:05:46 +02:00
parent 97bb2325df
commit ef162655af
3 changed files with 129 additions and 79 deletions

View File

@ -32,7 +32,7 @@ maint-checks:
stage: check stage: check
image: debian:stable-slim image: debian:stable-slim
script: script:
- apt-get update && apt-get install -y python3-toml python-is-python3 gawk - apt-get update && apt-get install -y python3-toml python-is-python3
- ./maint/check_toposort - ./maint/check_toposort
- ./maint/add_warning --check - ./maint/add_warning --check
- ./maint/check_doc_features - ./maint/check_doc_features

View File

@ -1,94 +1,143 @@
#!/bin/bash #!/usr/bin/python
set -euo pipefail import os
from list_crates import crate_list
from collections import Counter
MAINT=$(dirname "$0") # This contains annotations the script think are missing, but actually they don't need to be there
additional_provided = {}
# This contains annotations the script detected and think shouldn't be there, but actually they should
additional_required = {}
# annotation is provided in a way this script doesn't understand
declare -A provided
provided=()
# annotation is required for reasons this script doesn't understand
declare -A additional_required
additional_required=()
# PreferredRuntime has a somewhat more complexe rule for existing # PreferredRuntime has a somewhat more complexe rule for existing
provided[tor-rtcompat]=' additional_provided['tor-rtcompat'] = [
PreferredRuntime:all(feature = "native-tls") ('PreferredRuntime', 'all(feature = "native-tls")'),
PreferredRuntime:all(feature = "rustls", not(feature = "native-tls")) ('PreferredRuntime', 'all(feature = "rustls", not(feature = "native-tls"))'),
PreferredRuntime:feature = "native-tls" ('PreferredRuntime', 'feature = "native-tls"'),
PreferredRuntime:all(feature = "rustls", not(feature = "native-tls")) ('PreferredRuntime', 'all(feature = "rustls", not(feature = "native-tls"))'),
' ]
# Sha1 is present both ways # Sha1 is present both ways
provided[tor-llcrypto]=' additional_provided['tor-llcrypto'] = [
Sha1:feature = "with-openssl" ('Sha1', 'feature = "with-openssl"'),
Sha1:not(feature = "with-openssl") ('Sha1', 'not(feature = "with-openssl")'),
' ]
additional_required[tor-llcrypto]=' additional_required['tor-llcrypto']= [
aes:all() ('aes', 'all()'),
aes:all() ('aes', 'all()'),
' ]
# This is an * include; corresponding elements added to additional_required # This is an * include; expended wildcard must be in additional_required
provided[tor-proto]=' additional_provided['tor-proto']= [
*:feature = "testing" ('*', 'feature = "testing"'),
' ]
additional_required[tor-proto]=' additional_required['tor-proto']= [
CtrlMsg:feature = "testing" ('CtrlMsg', 'feature = "testing"'),
CreateResponse:feature = "testing" ('CreateResponse', 'feature = "testing"'),
' ]
# This is detected two times, but only on cfg_attr(docsrs) is enough # This is detected two times, but only on cfg_attr(docsrs) is enough
provided[tor-netdoc]=' additional_provided['tor-netdoc']= [
Nickname:feature = "dangerous-expose-struct-fields" ('Nickname', 'feature = "dangerous-expose-struct-fields"'),
NsConsensusRouterStatus:feature = "ns_consensus" ('NsConsensusRouterStatus', 'feature = "ns_consensus"'),
' ]
for crate in $("$MAINT/list_crates"); do
echo "checking $crate"
required=$( (find "crates/$crate" -type f -name '*.rs' -exec awk '
/^\s*pub use/ {
match($0, /^\s*pub use.*[ :]([^: ]+)(;| \{)/, arr)
itemName = arr[1]
if(cfg != "") {
print itemName ":" cfg
}
}
{ def extract_feature_pub_use(path):
match($0, /#\[cfg\((.*)\)\]/, arr) START_CFG = '#[cfg('
if(arr[1] != "") { END_CFG = ')]'
cfg = arr[1] PUB_USE = 'pub use '
} else { res = []
cfg = "" cfg = None
} with open(path, 'r') as file:
} for line in file.readlines():
' '{}' \;;echo "${additional_required["$crate"]:-}") | sed '/^$/d' | sort) if line.find(PUB_USE) != -1 and cfg:
# last line was a #[cfg(..)] line and this is a pub use
pubuse_pos = line.find(PUB_USE)
# ignore comments
if '//' in line[:pubuse_pos]:
continue
# extract ident
start = line.rfind(":")
if start == -1:
start = line.rfind(" ")
ident = line[start + 1:]
if pos:= ident.find(';'):
ident = ident[:pos]
res.append((ident, cfg))
cfg = None
continue
found=$( (find "crates/$crate" -type f -name '*.rs' -exec awk ' # check if we are on a #[cfg(..)] line, if so, remember it
/(struct|enum|mod|trait) / { start_cfg = line.find(START_CFG)
match($0, /(struct|enum|mod|trait) (\w*)(<.*>)?( \{|\()/, arr) end_cfg = line.find(END_CFG)
itemName = arr[2] if start_cfg == -1 or end_cfg == -1:
if(cfg != "") { cfg = None
print itemName ":" cfg else:
} start_cfg += len(START_CFG)
} cfg = line[start_cfg:end_cfg]
{ return res
match($0, /#\[cfg_attr\(docsrs, doc\(cfg\((.*)\)\)\)\]/, arr)
if(arr[1] != "") {
cfg = arr[1]
} else if (!/^\s*#\[/) {
cfg = ""
}
}
' '{}' \;; echo "${provided["$crate"]:-}") | sed '/^$/d' | sort)
comm -23 <(echo "$required") <(echo "$found") | sed 's/^/feature but no cfg_attr(docsrs):/' def extract_cfg_attr(path):
comm -13 <(echo "$required") <(echo "$found") | sed 's/^/cfg_attr(docsrs) but no feature:/' START_CFG = '#[cfg_attr(docsrs, doc(cfg('
if comm -3 <(echo "$required") <(echo "$found") | grep -q '^' ; then END_CFG = ')))]'
echo "difference found" res = []
exit 1 cfg = None
fi with open(path, 'r') as file:
done for line in file.readlines():
pos = max([line.find(kw + ' ') for kw in ['struct', 'enum', 'mod', 'trait']])
if pos != -1 and cfg:
# last line was a cfg and this is a declaration
subline = line[pos:]
subline = subline[subline.find(' ') + 1:]
end = min(subline.find(pat) for pat in ' (<' if subline.find(pat) !=-1)
ident = subline[:end]
res.append((ident, cfg))
cfg = None
continue
# check if we are on a #[cfg_attr(docsrs, doc(cfg(..)))] line, if so, remember it
start_cfg = line.find(START_CFG)
end_cfg = line.find(END_CFG)
if start_cfg != -1 and end_cfg != -1:
start_cfg += len(START_CFG)
cfg = line[start_cfg:end_cfg]
# don't reset when it's a cfg_attr followed by some other #[something]
elif "#[" not in line:
cfg = None
return res
def for_each_rs(path, fn):
res = []
for dir_, _, files in os.walk(path):
for file in files:
if not file.endswith(".rs"):
continue
res += fn(os.path.join(dir_, file))
return res
def main():
for crate in crate_list():
print(f"processing {crate}")
crate_path = f"crates/{crate}/src"
required = for_each_rs(crate_path, extract_feature_pub_use) + additional_required.get(crate, [])
provided = for_each_rs(crate_path, extract_cfg_attr) + additional_provided.get(crate, [])
req = Counter(required)
prov = Counter(provided)
ok = True
for elem in (req - prov).elements():
ok = False
print(f"feature but no cfg_attr(docsrs): {elem}")
for elem in (prov - req).elements():
ok = False
print(f"cfg_attr(docsrs) but no feature: {elem}")
if not ok:
print("Found error, exiting")
exit(1)
if __name__ == '__main__':
main()

1
maint/list_crates.py Symbolic link
View File

@ -0,0 +1 @@
list_crates