From 52ce9efb6aa9217fbf2b5a1191956e885d6956eb Mon Sep 17 00:00:00 2001 From: George Melikov Date: Wed, 7 Oct 2020 21:14:24 +0300 Subject: [PATCH] Add compatibility_matrix.py script as is Signed-off-by: George Melikov --- scripts/compatibility_matrix.py | 210 ++++++++++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 scripts/compatibility_matrix.py diff --git a/scripts/compatibility_matrix.py b/scripts/compatibility_matrix.py new file mode 100644 index 0000000..7f3605e --- /dev/null +++ b/scripts/compatibility_matrix.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 + +# License: CC0 https://creativecommons.org/share-your-work/public-domain/cc0/ + +# A messy script that figures out ZFS features. It's very messy, sorry. I am +# not responsible if this script eats your laundry. +# +# This uses manpages, because I'm lazy. If your manpages are wrong, you have a +# bug. +# +# If the script is wrong, or could be improved, feel free to contact me on +# freenode, and tell me why it's wrong. My nick is zgrep. +# +# 2018-07-05: Created. +# ????-??-??: Many things happened. +# 2020-02-23: Applied patch by rlaager. +# 2020-02-30: Show domain prefixes (via Vlad Bokov), partially apply patch by rlaager. +# 2020-03-05: Patch by rlaager (allocation_classes, ZoL -> openzfs). + +from sys import argv + +if len(argv) == 1: + path = '.' +elif len(argv) != 2: + print('Usage:', argv[0], 'path') + exit(1) +else: + path = argv[1] + +from collections import defaultdict +from urllib.request import urlopen +from datetime import datetime +from re import sub as regex, findall +from json import loads as dejson + +def zfsonlinux(): + sources = {'master':'https://raw.githubusercontent.com/openzfs/zfs/master/man/man5/zpool-features.5'} + with urlopen('https://zfsonlinux.org') as web: + versions = findall(r'download/zfs-([0-9.]+)', web.read().decode('utf-8', 'ignore')) + for ver in set(versions): + sources[ver] = 'https://raw.githubusercontent.com/openzfs/zfs/zfs-{}/man/man5/zpool-features.5'.format(ver) + return sources + +def openzfsonosx(): + sources = {'master': 'https://raw.githubusercontent.com/openzfsonosx/zfs/master/man/man5/zpool-features.5'} + with urlopen('https://api.github.com/repos/openzfsonosx/zfs/tags') as web: + try: + tags = dejson(web.read().decode('utf-8', 'ignore')) + tags = [ x['name'].lstrip('zfs-') for x in tags ] + tags.sort() + latest = tags[-1] + tags = [ tag for tag in tags if 'rc' not in tag ] + if 'rc' not in latest: + tags = tags[-3:] + else: + tags = tags[-2:] + [latest] + except: + tags = [] + for ver in tags: + sources[ver] = 'https://raw.githubusercontent.com/openzfsonosx/zfs/zfs-{}/man/man5/zpool-features.5'.format(ver) + return sources + +def freebsd(): + sources = {'head': 'https://svnweb.freebsd.org/base/head/cddl/contrib/opensolaris/cmd/zpool/zpool-features.7?view=co'} + with urlopen('https://www.freebsd.org/releases/') as web: + versions = findall(r'/releases/([0-9.]+?)R', web.read().decode('utf-8', 'ignore')) + with urlopen('https://svnweb.freebsd.org/base/release/') as web: + data = web.read().decode('utf-8', 'ignore') + actualversions = [] + for ver in set(versions): + found = list(sorted(findall( + r'/base/release/(' + ver.replace('.', '\\.') + r'[0-9.]*)', + data + ))) + if found: + actualversions.append(found[-1]) + for ver in actualversions: + sources[ver] = 'https://svnweb.freebsd.org/base/release/{}/cddl/contrib/opensolaris/cmd/zpool/zpool-features.7?view=co'.format(ver) + return sources + +def omniosce(): + sources = {'master': 'https://raw.githubusercontent.com/omniosorg/illumos-omnios/master/usr/src/man/man5/zpool-features.5'} + with urlopen('https://omniosce.org/releasenotes.html') as web: + versions = findall(r'omnios-build/blob/(r[0-9]+)', web.read().decode('utf-8', 'ignore')) + versions.sort() + versions = versions[-2:] + for ver in versions: + sources[ver] = 'https://raw.githubusercontent.com/omniosorg/illumos-omnios/{}/usr/src/man/man5/zpool-features.5'.format(ver) + return sources + +def joyent(): + sources = {'master': 'https://raw.githubusercontent.com/joyent/illumos-joyent/master/usr/src/man/man5/zpool-features.5'} + with urlopen('https://github.com/joyent/illumos-joyent') as web: + versions = findall(r'data-name="release-([0-9]+)"', web.read().decode('utf-8', 'ignore')) + versions.sort() + versions = versions[-2:] + for ver in versions: + sources[ver] = 'https://raw.githubusercontent.com/joyent/illumos-joyent/release-{}/usr/src/man/man5/zpool-features.5'.format(ver) + return sources + +def netbsd(): + url = 'http://cvsweb.netbsd.org/bsdweb.cgi/~checkout~/src/external/cddl/osnet/dist/cmd/zpool/zpool-features.7?content-type=text/plain&only_with_tag={}' + sources = { 'main': url.format('MAIN') } + with urlopen('https://netbsd.org/releases/') as web: + tags = findall(r'href="formal-.+?/NetBSD-(.+?)\.html', web.read().decode('utf-8', 'ignore')) + tags = [ (v, 'netbsd-' + v.replace('.', '-') + '-RELEASE') for v in tags ] + for ver, tag in tags: + if int(ver.split('.')[0]) >= 9: + sources[ver] = url.format(tag) + return sources + +sources = { + 'OpenZFS on Linux': zfsonlinux(), + 'FreeBSD': freebsd(), + 'OpenZFS on OS X': openzfsonosx(), + 'OmniOS CE': omniosce(), + 'Joyent': joyent(), + 'NetBSD': netbsd(), + 'Illumos': { + 'master': 'https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/man/man5/zpool-features.5', + }, +# 'OpenZFS on Windows': { +# 'master': 'https://raw.githubusercontent.com/openzfsonwindows/ZFSin/master/ZFSin/zfs/man/man5/zpool-features.5', +# }, + } + +features = defaultdict(list) +readonly = dict() + +for name, sub in sources.items(): + for ver, url in sub.items(): + with urlopen(url) as c: + if c.getcode() != 200: + continue + man = c.read().decode('utf-8') + for line in man.split('\n'): + if line.startswith('.It '): + line = line[4:] + if line.startswith('GUID'): + guid = line.split()[-1] + if guid == 'com.intel:allocation_classes': + # This is wrong in the documentation for Illumos and + # FreeBSD. The actual code in zfeature_common.c uses + # org.zfsonlinux:allocation_classes. + guid = 'org.zfsonlinux:allocation_classes' + elif guid == 'org.open-zfs:large_block': + guid += 's' + domain, feature = guid.split(':', 1) + features[(feature, domain)].append((name, ver)) + elif line.startswith('READ\\-ONLY COMPATIBLE'): + readonly[guid] = (line.split()[-1] == 'yes') + +header = list(sorted(sources.keys())) +header = list(zip(header, (sorted(sources[name], + key=lambda x: regex(r'[^0-9]', '', x) or x) for name in header))) +header.append(('Sortix', ('current',))) + +html = open(path + '/zfs.html', 'w') + +f_len, d_len = zip(*features.keys()) +f_len, d_len = max(map(len, f_len)), max(map(len, d_len)) + 1 + +html.write(''' +ZFS Feature Matrix + + + +''') + +html.write('\n') +html.write('') +html.write('') + +for name, vers in header: + html.write('') +html.write('\n') +for _, vers in header: + for ver in vers: + html.write('') +html.write('\n') + +for (feature, domain), names in sorted(features.items()): + guid = domain + ':' + feature + html.write(f'') + if readonly[guid]: + html.write('') + else: + html.write('') + for name, vers in header: + for ver in vers: + if (name, ver) in names: + html.write('') + else: + html.write('') + html.write('\n') +html.write('
Feature FlagRead-Only
Compatible		' + name + '
' + ver + '
{domain}:{feature}yesnoyesno
\n') + +now = datetime.now().isoformat() + 'Z' +html.write('

This works by parsing manpages for feature flags, and is entirely dependent on good, accurate documentation.
Last updated on ' + now + ' using zfs.py.

\n') + +html.close()