switch from awk to python
This commit is contained in:
parent
97bb2325df
commit
ef162655af
|
@ -32,7 +32,7 @@ maint-checks:
|
||||||
stage: check
|
stage: check
|
||||||
image: debian:stable-slim
|
image: debian:stable-slim
|
||||||
script:
|
script:
|
||||||
- apt-get update && apt-get install -y python3-toml python-is-python3 gawk
|
- apt-get update && apt-get install -y python3-toml python-is-python3
|
||||||
- ./maint/check_toposort
|
- ./maint/check_toposort
|
||||||
- ./maint/add_warning --check
|
- ./maint/add_warning --check
|
||||||
- ./maint/check_doc_features
|
- ./maint/check_doc_features
|
||||||
|
|
|
@ -1,94 +1,143 @@
|
||||||
#!/bin/bash
|
#!/usr/bin/python
|
||||||
|
|
||||||
set -euo pipefail
|
import os
|
||||||
|
from list_crates import crate_list
|
||||||
|
from collections import Counter
|
||||||
|
|
||||||
MAINT=$(dirname "$0")
|
# This contains annotations the script think are missing, but actually they don't need to be there
|
||||||
|
additional_provided = {}
|
||||||
|
|
||||||
|
# This contains annotations the script detected and think shouldn't be there, but actually they should
|
||||||
|
additional_required = {}
|
||||||
|
|
||||||
# annotation is provided in a way this script doesn't understand
|
|
||||||
declare -A provided
|
|
||||||
provided=()
|
|
||||||
# annotation is required for reasons this script doesn't understand
|
|
||||||
declare -A additional_required
|
|
||||||
additional_required=()
|
|
||||||
|
|
||||||
# PreferredRuntime has a somewhat more complexe rule for existing
|
# PreferredRuntime has a somewhat more complexe rule for existing
|
||||||
provided[tor-rtcompat]='
|
additional_provided['tor-rtcompat'] = [
|
||||||
PreferredRuntime:all(feature = "native-tls")
|
('PreferredRuntime', 'all(feature = "native-tls")'),
|
||||||
PreferredRuntime:all(feature = "rustls", not(feature = "native-tls"))
|
('PreferredRuntime', 'all(feature = "rustls", not(feature = "native-tls"))'),
|
||||||
PreferredRuntime:feature = "native-tls"
|
('PreferredRuntime', 'feature = "native-tls"'),
|
||||||
PreferredRuntime:all(feature = "rustls", not(feature = "native-tls"))
|
('PreferredRuntime', 'all(feature = "rustls", not(feature = "native-tls"))'),
|
||||||
'
|
]
|
||||||
|
|
||||||
# Sha1 is present both ways
|
# Sha1 is present both ways
|
||||||
provided[tor-llcrypto]='
|
additional_provided['tor-llcrypto'] = [
|
||||||
Sha1:feature = "with-openssl"
|
('Sha1', 'feature = "with-openssl"'),
|
||||||
Sha1:not(feature = "with-openssl")
|
('Sha1', 'not(feature = "with-openssl")'),
|
||||||
'
|
]
|
||||||
|
|
||||||
additional_required[tor-llcrypto]='
|
additional_required['tor-llcrypto']= [
|
||||||
aes:all()
|
('aes', 'all()'),
|
||||||
aes:all()
|
('aes', 'all()'),
|
||||||
'
|
]
|
||||||
|
|
||||||
# This is an * include; corresponding elements added to additional_required
|
# This is an * include; expended wildcard must be in additional_required
|
||||||
provided[tor-proto]='
|
additional_provided['tor-proto']= [
|
||||||
*:feature = "testing"
|
('*', 'feature = "testing"'),
|
||||||
'
|
]
|
||||||
|
|
||||||
additional_required[tor-proto]='
|
additional_required['tor-proto']= [
|
||||||
CtrlMsg:feature = "testing"
|
('CtrlMsg', 'feature = "testing"'),
|
||||||
CreateResponse:feature = "testing"
|
('CreateResponse', 'feature = "testing"'),
|
||||||
'
|
]
|
||||||
|
|
||||||
# This is detected two times, but only on cfg_attr(docsrs) is enough
|
# This is detected two times, but only on cfg_attr(docsrs) is enough
|
||||||
provided[tor-netdoc]='
|
additional_provided['tor-netdoc']= [
|
||||||
Nickname:feature = "dangerous-expose-struct-fields"
|
('Nickname', 'feature = "dangerous-expose-struct-fields"'),
|
||||||
NsConsensusRouterStatus:feature = "ns_consensus"
|
('NsConsensusRouterStatus', 'feature = "ns_consensus"'),
|
||||||
'
|
]
|
||||||
|
|
||||||
for crate in $("$MAINT/list_crates"); do
|
|
||||||
echo "checking $crate"
|
|
||||||
required=$( (find "crates/$crate" -type f -name '*.rs' -exec awk '
|
|
||||||
/^\s*pub use/ {
|
|
||||||
match($0, /^\s*pub use.*[ :]([^: ]+)(;| \{)/, arr)
|
|
||||||
itemName = arr[1]
|
|
||||||
if(cfg != "") {
|
|
||||||
print itemName ":" cfg
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
{
|
def extract_feature_pub_use(path):
|
||||||
match($0, /#\[cfg\((.*)\)\]/, arr)
|
START_CFG = '#[cfg('
|
||||||
if(arr[1] != "") {
|
END_CFG = ')]'
|
||||||
cfg = arr[1]
|
PUB_USE = 'pub use '
|
||||||
} else {
|
res = []
|
||||||
cfg = ""
|
cfg = None
|
||||||
}
|
with open(path, 'r') as file:
|
||||||
}
|
for line in file.readlines():
|
||||||
' '{}' \;;echo "${additional_required["$crate"]:-}") | sed '/^$/d' | sort)
|
if line.find(PUB_USE) != -1 and cfg:
|
||||||
|
# last line was a #[cfg(..)] line and this is a pub use
|
||||||
|
pubuse_pos = line.find(PUB_USE)
|
||||||
|
# ignore comments
|
||||||
|
if '//' in line[:pubuse_pos]:
|
||||||
|
continue
|
||||||
|
# extract ident
|
||||||
|
start = line.rfind(":")
|
||||||
|
if start == -1:
|
||||||
|
start = line.rfind(" ")
|
||||||
|
ident = line[start + 1:]
|
||||||
|
if pos:= ident.find(';'):
|
||||||
|
ident = ident[:pos]
|
||||||
|
res.append((ident, cfg))
|
||||||
|
cfg = None
|
||||||
|
continue
|
||||||
|
|
||||||
found=$( (find "crates/$crate" -type f -name '*.rs' -exec awk '
|
# check if we are on a #[cfg(..)] line, if so, remember it
|
||||||
/(struct|enum|mod|trait) / {
|
start_cfg = line.find(START_CFG)
|
||||||
match($0, /(struct|enum|mod|trait) (\w*)(<.*>)?( \{|\()/, arr)
|
end_cfg = line.find(END_CFG)
|
||||||
itemName = arr[2]
|
if start_cfg == -1 or end_cfg == -1:
|
||||||
if(cfg != "") {
|
cfg = None
|
||||||
print itemName ":" cfg
|
else:
|
||||||
}
|
start_cfg += len(START_CFG)
|
||||||
}
|
cfg = line[start_cfg:end_cfg]
|
||||||
{
|
return res
|
||||||
match($0, /#\[cfg_attr\(docsrs, doc\(cfg\((.*)\)\)\)\]/, arr)
|
|
||||||
if(arr[1] != "") {
|
|
||||||
cfg = arr[1]
|
|
||||||
} else if (!/^\s*#\[/) {
|
|
||||||
cfg = ""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
' '{}' \;; echo "${provided["$crate"]:-}") | sed '/^$/d' | sort)
|
|
||||||
|
|
||||||
comm -23 <(echo "$required") <(echo "$found") | sed 's/^/feature but no cfg_attr(docsrs):/'
|
def extract_cfg_attr(path):
|
||||||
comm -13 <(echo "$required") <(echo "$found") | sed 's/^/cfg_attr(docsrs) but no feature:/'
|
START_CFG = '#[cfg_attr(docsrs, doc(cfg('
|
||||||
if comm -3 <(echo "$required") <(echo "$found") | grep -q '^' ; then
|
END_CFG = ')))]'
|
||||||
echo "difference found"
|
res = []
|
||||||
exit 1
|
cfg = None
|
||||||
fi
|
with open(path, 'r') as file:
|
||||||
done
|
for line in file.readlines():
|
||||||
|
pos = max([line.find(kw + ' ') for kw in ['struct', 'enum', 'mod', 'trait']])
|
||||||
|
if pos != -1 and cfg:
|
||||||
|
# last line was a cfg and this is a declaration
|
||||||
|
subline = line[pos:]
|
||||||
|
subline = subline[subline.find(' ') + 1:]
|
||||||
|
end = min(subline.find(pat) for pat in ' (<' if subline.find(pat) !=-1)
|
||||||
|
ident = subline[:end]
|
||||||
|
res.append((ident, cfg))
|
||||||
|
cfg = None
|
||||||
|
continue
|
||||||
|
|
||||||
|
# check if we are on a #[cfg_attr(docsrs, doc(cfg(..)))] line, if so, remember it
|
||||||
|
start_cfg = line.find(START_CFG)
|
||||||
|
end_cfg = line.find(END_CFG)
|
||||||
|
if start_cfg != -1 and end_cfg != -1:
|
||||||
|
start_cfg += len(START_CFG)
|
||||||
|
cfg = line[start_cfg:end_cfg]
|
||||||
|
# don't reset when it's a cfg_attr followed by some other #[something]
|
||||||
|
elif "#[" not in line:
|
||||||
|
cfg = None
|
||||||
|
return res
|
||||||
|
|
||||||
|
def for_each_rs(path, fn):
|
||||||
|
res = []
|
||||||
|
for dir_, _, files in os.walk(path):
|
||||||
|
for file in files:
|
||||||
|
if not file.endswith(".rs"):
|
||||||
|
continue
|
||||||
|
res += fn(os.path.join(dir_, file))
|
||||||
|
return res
|
||||||
|
|
||||||
|
def main():
|
||||||
|
for crate in crate_list():
|
||||||
|
print(f"processing {crate}")
|
||||||
|
crate_path = f"crates/{crate}/src"
|
||||||
|
required = for_each_rs(crate_path, extract_feature_pub_use) + additional_required.get(crate, [])
|
||||||
|
provided = for_each_rs(crate_path, extract_cfg_attr) + additional_provided.get(crate, [])
|
||||||
|
req = Counter(required)
|
||||||
|
prov = Counter(provided)
|
||||||
|
ok = True
|
||||||
|
for elem in (req - prov).elements():
|
||||||
|
ok = False
|
||||||
|
print(f"feature but no cfg_attr(docsrs): {elem}")
|
||||||
|
for elem in (prov - req).elements():
|
||||||
|
ok = False
|
||||||
|
print(f"cfg_attr(docsrs) but no feature: {elem}")
|
||||||
|
if not ok:
|
||||||
|
print("Found error, exiting")
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
|
|
|
@ -0,0 +1 @@
|
||||||
|
list_crates
|
Loading…
Reference in New Issue