Created
April 16, 2025 19:36
-
-
Save minrk/d341a340f771a9223fda2914f4f52e87 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "cells": [ | |
| { | |
| "cell_type": "markdown", | |
| "id": "1f58579c-398b-461b-8aa7-c99c8f67485a", | |
| "metadata": {}, | |
| "source": [ | |
| "# Collect info on jupyterhub packages\n", | |
| "\n", | |
| "Looks for repos that publish Python ackages\n", | |
| "\n", | |
| "GitHub:\n", | |
| "\n", | |
| "- List all repos with GraphQL\n", | |
| "- Get package name from pyproject.toml\n", | |
| "- For a few older packages, look in setup.cfg and setup.py, if no pyproject.toml\n", | |
| "\n", | |
| "For each found package, look it up on PyPI:\n", | |
| "\n", | |
| "- Check if packages exist on PyPI\n", | |
| " " | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 1, | |
| "id": "d388ed30-79ce-4419-ae86-d21ec34da66c", | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "Using netrc auth token\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "import json\n", | |
| "import netrc\n", | |
| "import re\n", | |
| "import configparser\n", | |
| "from configparser import ConfigParser\n", | |
| "\n", | |
| "from jinja2 import Template\n", | |
| "import requests\n", | |
| "import requests_cache\n", | |
| "import toml\n", | |
| "\n", | |
| "\n", | |
| "file_gql = Template(\n", | |
| " \"\"\"\n", | |
| "{\n", | |
| " organization(login: \"{{ organization }}\") {\n", | |
| " repositories(first: 100{% if after -%}, after: \"{{ after }}\" {%- endif %}) {\n", | |
| " pageInfo {\n", | |
| " endCursor\n", | |
| " hasNextPage\n", | |
| " }\n", | |
| " edges {\n", | |
| " node {\n", | |
| " isArchived\n", | |
| " nameWithOwner\n", | |
| " object(expression: \"HEAD:{{ path }}\") {\n", | |
| " ... on Blob {\n", | |
| " text\n", | |
| " }\n", | |
| " }\n", | |
| " }\n", | |
| " }\n", | |
| "\n", | |
| " }\n", | |
| " }\n", | |
| "}\n", | |
| "\"\"\"\n", | |
| ")\n", | |
| "\n", | |
| "requests_cache.install_cache(\"github\", allowable_methods=[\"GET\", \"POST\"])\n", | |
| "\n", | |
| "s = requests.Session()\n", | |
| "auth = netrc.netrc().authenticators(\"api.github.com\")\n", | |
| "if auth:\n", | |
| " print(\"Using netrc auth token\")\n", | |
| " s.headers[\"Authorization\"] = f\"bearer {auth[2]}\"\n", | |
| "else:\n", | |
| " print(\"No auth\")\n", | |
| "\n", | |
| "github_graphql = \"https://api.github.com/graphql\"" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 2, | |
| "id": "ec6b37e6-b538-47ca-820e-1ab85817705d", | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "def fetch_files(path, organization: str = \"jupyterhub\", after=None):\n", | |
| " query = file_gql.render(\n", | |
| " organization=organization,\n", | |
| " after=after,\n", | |
| " path=path,\n", | |
| " )\n", | |
| " resp = s.post(github_graphql, data=json.dumps(dict(query=query)))\n", | |
| " result = resp.json()\n", | |
| " if resp.status_code >= 400:\n", | |
| " print(result)\n", | |
| " resp.raise_for_status()\n", | |
| " repos = result[\"data\"][\"organization\"][\"repositories\"]\n", | |
| " for repo_edge in repos[\"edges\"]:\n", | |
| " repo = repo_edge[\"node\"]\n", | |
| " if repo[\"isArchived\"]:\n", | |
| " # ignore archived repos\n", | |
| " continue\n", | |
| "\n", | |
| " repo_info = {\n", | |
| " \"name\": repo[\"nameWithOwner\"],\n", | |
| " path: None,\n", | |
| " }\n", | |
| " if repo[\"object\"]:\n", | |
| " repo_info[path] = repo[\"object\"][\"text\"]\n", | |
| " yield repo_info\n", | |
| " # pagination\n", | |
| " if repos[\"pageInfo\"][\"hasNextPage\"]:\n", | |
| " yield from fetch_files(\n", | |
| " path,\n", | |
| " organization=organization,\n", | |
| " after=repos[\"pageInfo\"][\"endCursor\"],\n", | |
| " )\n", | |
| "\n", | |
| "\n", | |
| "def name_from_pyproject_toml(pyproject_toml):\n", | |
| " \"\"\"get package name from pyproject.toml\"\"\"\n", | |
| " pyproject = toml.loads(pyproject_toml)\n", | |
| " return pyproject.get(\"project\", {}).get(\"name\")\n", | |
| "\n", | |
| "\n", | |
| "def name_from_setup_cfg(setup_cfg):\n", | |
| " cfg = ConfigParser()\n", | |
| " cfg.read_string(setup_cfg)\n", | |
| " try:\n", | |
| " return cfg.get(\"metadata\", \"name\")\n", | |
| " except configparser.Error:\n", | |
| " return None\n", | |
| "\n", | |
| "\n", | |
| "def name_from_setup_py(setup_py):\n", | |
| " \"\"\"get package name from setup.py\"\"\"\n", | |
| " m = None\n", | |
| " for name in re.findall(r\"\"\"name\\s*=\\s*['\"]([^'\"]+)['\"]\"\"\", setup_py):\n", | |
| " # this is a regex, but probably good enough for a few old setup.pys\n", | |
| " return name\n", | |
| " return m\n", | |
| "\n", | |
| "\n", | |
| "def collect_repo_packages(organizations, verbose=False):\n", | |
| " repos = {}\n", | |
| " for organization in organizations:\n", | |
| " for repo in fetch_files(\"pyproject.toml\", organization):\n", | |
| " name = repo[\"name\"]\n", | |
| " repo[\"package\"] = None\n", | |
| " repos[name] = repo\n", | |
| " pyproject_toml = repo[\"pyproject.toml\"]\n", | |
| " if pyproject_toml:\n", | |
| " repo[\"package\"] = pkg = name_from_pyproject_toml(pyproject_toml)\n", | |
| " else:\n", | |
| " if verbose:\n", | |
| " print(f\"{name}: no pyproject.toml\")\n", | |
| " for new_info in fetch_files(\"setup.cfg\", organization):\n", | |
| " name = new_info[\"name\"]\n", | |
| " repo = repos[name]\n", | |
| " repo.update(new_info)\n", | |
| " if repo[\"setup.cfg\"] and not repo[\"package\"]:\n", | |
| " repo[\"package\"] = pkg = name_from_setup_cfg(repo[\"setup.cfg\"])\n", | |
| " if pkg and verbose:\n", | |
| " print(f\"{name}: got name from setup.cfg\")\n", | |
| " for new_info in fetch_files(\"setup.py\", organization):\n", | |
| " name = new_info[\"name\"]\n", | |
| " repo = repos[name]\n", | |
| " repo.update(new_info)\n", | |
| " if repo[\"setup.py\"] and not repo[\"package\"]:\n", | |
| " repo[\"package\"] = pkg = name_from_setup_py(repo[\"setup.py\"])\n", | |
| " if pkg and verbose:\n", | |
| " print(f\"{name}: got name from setup.py\")\n", | |
| "\n", | |
| " if verbose:\n", | |
| " print(\"----\")\n", | |
| " for repo in repos.values():\n", | |
| " print(f\"{repo['name']}: {repo['package']}\")\n", | |
| " return repos\n", | |
| "\n", | |
| "\n", | |
| "repos = collect_repo_packages([\"jupyterhub\", \"jupyterhealth\"])" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "id": "f7bd0d97-e23a-4065-a4c4-93e51ef9fb0c", | |
| "metadata": {}, | |
| "source": [ | |
| "Scrape members of Jupyter org. No API for this?" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 3, | |
| "id": "ec095f32-703c-4c3a-bc9d-74c367bf4527", | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "from bs4 import BeautifulSoup\n", | |
| "\n", | |
| "jupyter_org_url = \"https://pypi.org/org/jupyter/\"\n", | |
| "r = requests.get(jupyter_org_url)\n", | |
| "r.raise_for_status()\n", | |
| "page = BeautifulSoup(r.text)\n", | |
| "\n", | |
| "jupyter_org_packages = {\n", | |
| " el.text.strip() for el in page.find_all(class_=\"package-snippet__title\")\n", | |
| "}" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "id": "56c5386c-8a5f-4a0c-84d4-b3bc4827c0ff", | |
| "metadata": {}, | |
| "source": [ | |
| "Get information from PyPI:" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 4, | |
| "id": "eb3422e2-e14f-4e4b-8016-3ffac19a641d", | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "def get_pypi_info(repo):\n", | |
| " package = repo[\"package\"]\n", | |
| " if not package:\n", | |
| " return repo\n", | |
| " repo[\"pypi_url\"] = None\n", | |
| " repo[\"pypi_version\"] = None\n", | |
| " repo[\"trusted_publisher\"] = None\n", | |
| " pypi_url = f\"https://pypi.org/pypi/{package}/json\"\n", | |
| " r = requests.get(pypi_url)\n", | |
| " if r.status_code == 404:\n", | |
| " return repo\n", | |
| " if not r.ok:\n", | |
| " print(f\"{r.status_code}: {pypi_url} (not a published package?)\")\n", | |
| " return repo\n", | |
| " pypi_info = r.json()\n", | |
| " repo[\"package\"] = package = pypi_info[\"info\"][\"name\"] # apply PyPI normalization\n", | |
| " repo[\"jupyter_org\"] = package in jupyter_org_packages\n", | |
| " repo[\"pypi_url\"] = pypi_info[\"info\"][\"package_url\"]\n", | |
| " repo[\"pypi_version\"] = version = pypi_info[\"info\"][\"version\"]\n", | |
| " filename = pypi_info[\"releases\"][version][0][\"filename\"]\n", | |
| " provenance_url = (\n", | |
| " f\"https://pypi.org/integrity/{package}/{version}/{filename}/provenance\"\n", | |
| " )\n", | |
| " r = requests.get(provenance_url)\n", | |
| " if r.ok:\n", | |
| " repo[\"trusted_publisher\"] = True\n", | |
| " else:\n", | |
| " try:\n", | |
| " r.json()\n", | |
| " except:\n", | |
| " # not JSON?\n", | |
| " print(provenance_url)\n", | |
| " print(r.text)\n", | |
| " raise\n", | |
| " else:\n", | |
| " repo[\"trusted_publisher\"] = False\n", | |
| " return repo\n", | |
| "\n", | |
| "\n", | |
| "for repo in repos.values():\n", | |
| " get_pypi_info(repo)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 5, | |
| "id": "dcdfc3a4-09bf-4de1-8153-5579e29a1b31", | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": [ | |
| "<div>\n", | |
| "<style scoped>\n", | |
| " .dataframe tbody tr th:only-of-type {\n", | |
| " vertical-align: middle;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe tbody tr th {\n", | |
| " vertical-align: top;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe thead th {\n", | |
| " text-align: right;\n", | |
| " }\n", | |
| "</style>\n", | |
| "<table border=\"1\" class=\"dataframe\">\n", | |
| " <thead>\n", | |
| " <tr style=\"text-align: right;\">\n", | |
| " <th></th>\n", | |
| " <th>name</th>\n", | |
| " <th>pyproject.toml</th>\n", | |
| " <th>package</th>\n", | |
| " <th>setup.cfg</th>\n", | |
| " <th>setup.py</th>\n", | |
| " <th>pypi_url</th>\n", | |
| " <th>pypi_version</th>\n", | |
| " <th>trusted_publisher</th>\n", | |
| " <th>jupyter_org</th>\n", | |
| " </tr>\n", | |
| " </thead>\n", | |
| " <tbody>\n", | |
| " <tr>\n", | |
| " <th>0</th>\n", | |
| " <td>jupyterhub/jupyterhub</td>\n", | |
| " <td># PEP 621 build info\\n[build-system]\\nrequires...</td>\n", | |
| " <td>jupyterhub</td>\n", | |
| " <td>None</td>\n", | |
| " <td>#!/usr/bin/env python3\\n# Copyright (c) Jupyte...</td>\n", | |
| " <td>https://pypi.org/project/jupyterhub/</td>\n", | |
| " <td>5.3.0</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>1</th>\n", | |
| " <td>jupyterhub/configurable-http-proxy</td>\n", | |
| " <td>None</td>\n", | |
| " <td>None</td>\n", | |
| " <td>None</td>\n", | |
| " <td>None</td>\n", | |
| " <td>NaN</td>\n", | |
| " <td>NaN</td>\n", | |
| " <td>NaN</td>\n", | |
| " <td>NaN</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>2</th>\n", | |
| " <td>jupyterhub/oauthenticator</td>\n", | |
| " <td># autoflake is used for autoformatting Python ...</td>\n", | |
| " <td>oauthenticator</td>\n", | |
| " <td>None</td>\n", | |
| " <td>#!/usr/bin/env python\\n# Copyright (c) Jupyter...</td>\n", | |
| " <td>https://pypi.org/project/oauthenticator/</td>\n", | |
| " <td>17.3.0</td>\n", | |
| " <td>True</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>3</th>\n", | |
| " <td>jupyterhub/dockerspawner</td>\n", | |
| " <td># autoflake is used for autoformatting Python ...</td>\n", | |
| " <td>dockerspawner</td>\n", | |
| " <td>None</td>\n", | |
| " <td>#!/usr/bin/env python\\n# Copyright (c) Jupyter...</td>\n", | |
| " <td>https://pypi.org/project/dockerspawner/</td>\n", | |
| " <td>14.0.0</td>\n", | |
| " <td>True</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>4</th>\n", | |
| " <td>jupyterhub/sudospawner</td>\n", | |
| " <td>None</td>\n", | |
| " <td>sudospawner</td>\n", | |
| " <td>None</td>\n", | |
| " <td>#!/usr/bin/env python\\n# coding: utf-8\\n\\n# Co...</td>\n", | |
| " <td>https://pypi.org/project/sudospawner/</td>\n", | |
| " <td>0.5.2</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>...</th>\n", | |
| " <td>...</td>\n", | |
| " <td>...</td>\n", | |
| " <td>...</td>\n", | |
| " <td>...</td>\n", | |
| " <td>...</td>\n", | |
| " <td>...</td>\n", | |
| " <td>...</td>\n", | |
| " <td>...</td>\n", | |
| " <td>...</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>74</th>\n", | |
| " <td>jupyterhealth/myst-site-article</td>\n", | |
| " <td>None</td>\n", | |
| " <td>None</td>\n", | |
| " <td>None</td>\n", | |
| " <td>None</td>\n", | |
| " <td>NaN</td>\n", | |
| " <td>NaN</td>\n", | |
| " <td>NaN</td>\n", | |
| " <td>NaN</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>75</th>\n", | |
| " <td>jupyterhealth/myst-site-book</td>\n", | |
| " <td>None</td>\n", | |
| " <td>None</td>\n", | |
| " <td>None</td>\n", | |
| " <td>None</td>\n", | |
| " <td>NaN</td>\n", | |
| " <td>NaN</td>\n", | |
| " <td>NaN</td>\n", | |
| " <td>NaN</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>76</th>\n", | |
| " <td>jupyterhealth/jupyterhealth-deploy</td>\n", | |
| " <td>None</td>\n", | |
| " <td>None</td>\n", | |
| " <td>None</td>\n", | |
| " <td>None</td>\n", | |
| " <td>NaN</td>\n", | |
| " <td>NaN</td>\n", | |
| " <td>NaN</td>\n", | |
| " <td>NaN</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>77</th>\n", | |
| " <td>jupyterhealth/jupyter-smart-on-fhir</td>\n", | |
| " <td>[build-system]\\nrequires = [\"setuptools>=77\", ...</td>\n", | |
| " <td>jupyter-smart-on-fhir</td>\n", | |
| " <td>None</td>\n", | |
| " <td>None</td>\n", | |
| " <td>https://pypi.org/project/jupyter-smart-on-fhir/</td>\n", | |
| " <td>0.1.0a3</td>\n", | |
| " <td>True</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>78</th>\n", | |
| " <td>jupyterhealth/jupyterhealth-client</td>\n", | |
| " <td>[project]\\nname = \"jupyterhealth-client\"\\ndesc...</td>\n", | |
| " <td>jupyterhealth-client</td>\n", | |
| " <td>None</td>\n", | |
| " <td>None</td>\n", | |
| " <td>https://pypi.org/project/jupyterhealth-client/</td>\n", | |
| " <td>0.0.1a4</td>\n", | |
| " <td>True</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " </tbody>\n", | |
| "</table>\n", | |
| "<p>79 rows × 9 columns</p>\n", | |
| "</div>" | |
| ], | |
| "text/plain": [ | |
| " name \\\n", | |
| "0 jupyterhub/jupyterhub \n", | |
| "1 jupyterhub/configurable-http-proxy \n", | |
| "2 jupyterhub/oauthenticator \n", | |
| "3 jupyterhub/dockerspawner \n", | |
| "4 jupyterhub/sudospawner \n", | |
| ".. ... \n", | |
| "74 jupyterhealth/myst-site-article \n", | |
| "75 jupyterhealth/myst-site-book \n", | |
| "76 jupyterhealth/jupyterhealth-deploy \n", | |
| "77 jupyterhealth/jupyter-smart-on-fhir \n", | |
| "78 jupyterhealth/jupyterhealth-client \n", | |
| "\n", | |
| " pyproject.toml package \\\n", | |
| "0 # PEP 621 build info\\n[build-system]\\nrequires... jupyterhub \n", | |
| "1 None None \n", | |
| "2 # autoflake is used for autoformatting Python ... oauthenticator \n", | |
| "3 # autoflake is used for autoformatting Python ... dockerspawner \n", | |
| "4 None sudospawner \n", | |
| ".. ... ... \n", | |
| "74 None None \n", | |
| "75 None None \n", | |
| "76 None None \n", | |
| "77 [build-system]\\nrequires = [\"setuptools>=77\", ... jupyter-smart-on-fhir \n", | |
| "78 [project]\\nname = \"jupyterhealth-client\"\\ndesc... jupyterhealth-client \n", | |
| "\n", | |
| " setup.cfg setup.py \\\n", | |
| "0 None #!/usr/bin/env python3\\n# Copyright (c) Jupyte... \n", | |
| "1 None None \n", | |
| "2 None #!/usr/bin/env python\\n# Copyright (c) Jupyter... \n", | |
| "3 None #!/usr/bin/env python\\n# Copyright (c) Jupyter... \n", | |
| "4 None #!/usr/bin/env python\\n# coding: utf-8\\n\\n# Co... \n", | |
| ".. ... ... \n", | |
| "74 None None \n", | |
| "75 None None \n", | |
| "76 None None \n", | |
| "77 None None \n", | |
| "78 None None \n", | |
| "\n", | |
| " pypi_url pypi_version \\\n", | |
| "0 https://pypi.org/project/jupyterhub/ 5.3.0 \n", | |
| "1 NaN NaN \n", | |
| "2 https://pypi.org/project/oauthenticator/ 17.3.0 \n", | |
| "3 https://pypi.org/project/dockerspawner/ 14.0.0 \n", | |
| "4 https://pypi.org/project/sudospawner/ 0.5.2 \n", | |
| ".. ... ... \n", | |
| "74 NaN NaN \n", | |
| "75 NaN NaN \n", | |
| "76 NaN NaN \n", | |
| "77 https://pypi.org/project/jupyter-smart-on-fhir/ 0.1.0a3 \n", | |
| "78 https://pypi.org/project/jupyterhealth-client/ 0.0.1a4 \n", | |
| "\n", | |
| " trusted_publisher jupyter_org \n", | |
| "0 False False \n", | |
| "1 NaN NaN \n", | |
| "2 True False \n", | |
| "3 True False \n", | |
| "4 False False \n", | |
| ".. ... ... \n", | |
| "74 NaN NaN \n", | |
| "75 NaN NaN \n", | |
| "76 NaN NaN \n", | |
| "77 True False \n", | |
| "78 True False \n", | |
| "\n", | |
| "[79 rows x 9 columns]" | |
| ] | |
| }, | |
| "execution_count": 5, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "import pandas as pd\n", | |
| "\n", | |
| "df = pd.DataFrame.from_records(list(repos.values()))\n", | |
| "df" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "id": "fba18064-a8fa-4eb3-b6eb-237e162daf66", | |
| "metadata": {}, | |
| "source": [ | |
| "Has package info, not on PyPI:" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 6, | |
| "id": "44d0585d-7c51-4aca-94c6-e49c2ab6e98a", | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "jupyterhub/jupyterhub-python-repo-template\n", | |
| "jupyterhub/nbgitpuller-downloader-plugins\n", | |
| "jupyterhub/the-littlest-jupyterhub\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "for repo in df.name[df.package.notna() & df.pypi_url.isna()].sort_values():\n", | |
| " print(repo)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "id": "d1b482b0-02ea-4b40-a1e3-5be490b42939", | |
| "metadata": {}, | |
| "source": [ | |
| "No Python package found:" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 7, | |
| "id": "d72947a5-a54d-4fb3-854d-9da6c071a0cb", | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "jupyterhealth/demos\n", | |
| "jupyterhealth/jupyter-health-software\n", | |
| "jupyterhealth/jupyterhealth-deploy\n", | |
| "jupyterhealth/jupyterhealth.github.io\n", | |
| "jupyterhealth/myst-site-article\n", | |
| "jupyterhealth/myst-site-book\n", | |
| "jupyterhealth/partner-client-docker\n", | |
| "jupyterhealth/singleuser-image\n", | |
| "jupyterhealth/software-documentation\n", | |
| "jupyterhub/.github\n", | |
| "jupyterhub/action-get-quayio-tags\n", | |
| "jupyterhub/action-k3s-helm\n", | |
| "jupyterhub/action-k8s-await-workloads\n", | |
| "jupyterhub/action-k8s-namespace-report\n", | |
| "jupyterhub/action-major-minor-tag-calculator\n", | |
| "jupyterhub/binder-billing\n", | |
| "jupyterhub/binder-data\n", | |
| "jupyterhub/configurable-http-proxy\n", | |
| "jupyterhub/design\n", | |
| "jupyterhub/grafana-dashboards\n", | |
| "jupyterhub/helm-chart\n", | |
| "jupyterhub/jupyterhub-container-images\n", | |
| "jupyterhub/jupyterhub-deploy-docker\n", | |
| "jupyterhub/jupyterhub-deploy-hpc\n", | |
| "jupyterhub/jupyterhub-deploy-teaching\n", | |
| "jupyterhub/jupyterhub-on-hadoop\n", | |
| "jupyterhub/jupyterhub-the-hard-way\n", | |
| "jupyterhub/jupyterhub-tutorial\n", | |
| "jupyterhub/jupyterhub.github.io\n", | |
| "jupyterhub/katacoda-scenarios\n", | |
| "jupyterhub/mybinder-tools\n", | |
| "jupyterhub/mybinder.org-deploy\n", | |
| "jupyterhub/mybinder.org-user-guide\n", | |
| "jupyterhub/outreachy\n", | |
| "jupyterhub/pebble-helm-chart\n", | |
| "jupyterhub/repo2docker-action\n", | |
| "jupyterhub/team-compass\n", | |
| "jupyterhub/zero-to-jupyterhub-k8s\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "for repo in df.name[df.package.isna()].sort_values():\n", | |
| " print(repo)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 8, | |
| "id": "74782d7b-2264-4812-bc96-b6f6c16a4a37", | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/plain": [ | |
| "38" | |
| ] | |
| }, | |
| "execution_count": 8, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "packages = df.loc[df.pypi_url.notna()]\n", | |
| "len(packages)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "id": "700d0e7b-7ed6-4bb7-b812-290f97fe238c", | |
| "metadata": {}, | |
| "source": [ | |
| "Repos with a package, but no pyproject.toml (should probably update or archive):" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 9, | |
| "id": "f8b5dec1-76f0-4294-a86c-8ada65606c74", | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "jupyterhub/firstuseauthenticator\n", | |
| "jupyterhub/gh-scoped-creds\n", | |
| "jupyterhub/kerberosauthenticator\n", | |
| "jupyterhub/nullauthenticator\n", | |
| "jupyterhub/repo2docker\n", | |
| "jupyterhub/sudospawner\n", | |
| "jupyterhub/wrapspawner\n", | |
| "jupyterhub/yarnspawner\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "for name in packages.name[packages[\"pyproject.toml\"].isna()].sort_values():\n", | |
| " print(name)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "id": "46394042-637a-4504-9cb9-ea17a85b2a95", | |
| "metadata": {}, | |
| "source": [ | |
| "## Package summary:" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 10, | |
| "id": "8800d78b-8189-4d23-8605-82bb5b3411a4", | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": [ | |
| "<div>\n", | |
| "<style scoped>\n", | |
| " .dataframe tbody tr th:only-of-type {\n", | |
| " vertical-align: middle;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe tbody tr th {\n", | |
| " vertical-align: top;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe thead th {\n", | |
| " text-align: right;\n", | |
| " }\n", | |
| "</style>\n", | |
| "<table border=\"1\" class=\"dataframe\">\n", | |
| " <thead>\n", | |
| " <tr style=\"text-align: right;\">\n", | |
| " <th></th>\n", | |
| " <th>name</th>\n", | |
| " <th>package</th>\n", | |
| " <th>jupyter_org</th>\n", | |
| " <th>trusted_publisher</th>\n", | |
| " </tr>\n", | |
| " </thead>\n", | |
| " <tbody>\n", | |
| " <tr>\n", | |
| " <th>0</th>\n", | |
| " <td>jupyterhub/batchspawner</td>\n", | |
| " <td>batchspawner</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>1</th>\n", | |
| " <td>jupyterhub/binderhub</td>\n", | |
| " <td>binderhub</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>2</th>\n", | |
| " <td>jupyterhub/chartpress</td>\n", | |
| " <td>chartpress</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>3</th>\n", | |
| " <td>jupyterhub/docker-image-cleaner</td>\n", | |
| " <td>docker-image-cleaner</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>4</th>\n", | |
| " <td>jupyterhub/gh-scoped-creds</td>\n", | |
| " <td>gh-scoped-creds</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>5</th>\n", | |
| " <td>jupyterhub/repo2docker</td>\n", | |
| " <td>jupyter-repo2docker</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>6</th>\n", | |
| " <td>jupyterhub/jupyter-server-proxy</td>\n", | |
| " <td>jupyter-server-proxy</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>7</th>\n", | |
| " <td>jupyterhub/jupyterhub</td>\n", | |
| " <td>jupyterhub</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>8</th>\n", | |
| " <td>jupyterhub/firstuseauthenticator</td>\n", | |
| " <td>jupyterhub-firstuseauthenticator</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>9</th>\n", | |
| " <td>jupyterhub/jupyterhub-idle-culler</td>\n", | |
| " <td>jupyterhub-idle-culler</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>10</th>\n", | |
| " <td>jupyterhub/kerberosauthenticator</td>\n", | |
| " <td>jupyterhub-kerberosauthenticator</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>11</th>\n", | |
| " <td>jupyterhub/kubespawner</td>\n", | |
| " <td>jupyterhub-kubespawner</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>12</th>\n", | |
| " <td>jupyterhub/ldapauthenticator</td>\n", | |
| " <td>jupyterhub-ldapauthenticator</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>13</th>\n", | |
| " <td>jupyterhub/ltiauthenticator</td>\n", | |
| " <td>jupyterhub-ltiauthenticator</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>14</th>\n", | |
| " <td>jupyterhub/nativeauthenticator</td>\n", | |
| " <td>jupyterhub-nativeauthenticator</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>15</th>\n", | |
| " <td>jupyterhub/jupyterhub-sphinx-theme</td>\n", | |
| " <td>jupyterhub-sphinx-theme</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>16</th>\n", | |
| " <td>jupyterhub/systemdspawner</td>\n", | |
| " <td>jupyterhub-systemdspawner</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>17</th>\n", | |
| " <td>jupyterhub/tmpauthenticator</td>\n", | |
| " <td>jupyterhub-tmpauthenticator</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>18</th>\n", | |
| " <td>jupyterhub/traefik-proxy</td>\n", | |
| " <td>jupyterhub-traefik-proxy</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>19</th>\n", | |
| " <td>jupyterhub/yarnspawner</td>\n", | |
| " <td>jupyterhub-yarnspawner</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>20</th>\n", | |
| " <td>jupyterhub/nbgitpuller-downloader-dropbox</td>\n", | |
| " <td>nbgitpuller-downloader-dropbox</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>21</th>\n", | |
| " <td>jupyterhub/nbgitpuller-downloader-generic-web</td>\n", | |
| " <td>nbgitpuller-downloader-generic-web</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>22</th>\n", | |
| " <td>jupyterhub/nbgitpuller-downloader-googledrive</td>\n", | |
| " <td>nbgitpuller-downloader-googledrive</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>23</th>\n", | |
| " <td>jupyterhub/nullauthenticator</td>\n", | |
| " <td>nullauthenticator</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>24</th>\n", | |
| " <td>jupyterhub/pytest-jupyterhub</td>\n", | |
| " <td>pytest-jupyterhub</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>25</th>\n", | |
| " <td>jupyterhub/simpervisor</td>\n", | |
| " <td>simpervisor</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>26</th>\n", | |
| " <td>jupyterhub/sudospawner</td>\n", | |
| " <td>sudospawner</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>27</th>\n", | |
| " <td>jupyterhub/wrapspawner</td>\n", | |
| " <td>wrapspawner</td>\n", | |
| " <td>False</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>28</th>\n", | |
| " <td>jupyterhub/dockerspawner</td>\n", | |
| " <td>dockerspawner</td>\n", | |
| " <td>False</td>\n", | |
| " <td>True</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>29</th>\n", | |
| " <td>jupyterhub/jupyter-remote-desktop-proxy</td>\n", | |
| " <td>jupyter-remote-desktop-proxy</td>\n", | |
| " <td>False</td>\n", | |
| " <td>True</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>30</th>\n", | |
| " <td>jupyterhub/jupyter-rsession-proxy</td>\n", | |
| " <td>jupyter-rsession-proxy</td>\n", | |
| " <td>False</td>\n", | |
| " <td>True</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>31</th>\n", | |
| " <td>jupyterhealth/jupyter-smart-on-fhir</td>\n", | |
| " <td>jupyter-smart-on-fhir</td>\n", | |
| " <td>False</td>\n", | |
| " <td>True</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>32</th>\n", | |
| " <td>jupyterhealth/jupyterhealth-client</td>\n", | |
| " <td>jupyterhealth-client</td>\n", | |
| " <td>False</td>\n", | |
| " <td>True</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>33</th>\n", | |
| " <td>jupyterhub/nbgitpuller</td>\n", | |
| " <td>nbgitpuller</td>\n", | |
| " <td>False</td>\n", | |
| " <td>True</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>34</th>\n", | |
| " <td>jupyterhub/oauthenticator</td>\n", | |
| " <td>oauthenticator</td>\n", | |
| " <td>False</td>\n", | |
| " <td>True</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>35</th>\n", | |
| " <td>jupyterhub/autodoc-traits</td>\n", | |
| " <td>autodoc-traits</td>\n", | |
| " <td>True</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>36</th>\n", | |
| " <td>jupyterhub/escapism</td>\n", | |
| " <td>escapism</td>\n", | |
| " <td>True</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>37</th>\n", | |
| " <td>jupyterhub/pamela</td>\n", | |
| " <td>pamela</td>\n", | |
| " <td>True</td>\n", | |
| " <td>False</td>\n", | |
| " </tr>\n", | |
| " </tbody>\n", | |
| "</table>\n", | |
| "</div>" | |
| ], | |
| "text/plain": [ | |
| " name \\\n", | |
| "0 jupyterhub/batchspawner \n", | |
| "1 jupyterhub/binderhub \n", | |
| "2 jupyterhub/chartpress \n", | |
| "3 jupyterhub/docker-image-cleaner \n", | |
| "4 jupyterhub/gh-scoped-creds \n", | |
| "5 jupyterhub/repo2docker \n", | |
| "6 jupyterhub/jupyter-server-proxy \n", | |
| "7 jupyterhub/jupyterhub \n", | |
| "8 jupyterhub/firstuseauthenticator \n", | |
| "9 jupyterhub/jupyterhub-idle-culler \n", | |
| "10 jupyterhub/kerberosauthenticator \n", | |
| "11 jupyterhub/kubespawner \n", | |
| "12 jupyterhub/ldapauthenticator \n", | |
| "13 jupyterhub/ltiauthenticator \n", | |
| "14 jupyterhub/nativeauthenticator \n", | |
| "15 jupyterhub/jupyterhub-sphinx-theme \n", | |
| "16 jupyterhub/systemdspawner \n", | |
| "17 jupyterhub/tmpauthenticator \n", | |
| "18 jupyterhub/traefik-proxy \n", | |
| "19 jupyterhub/yarnspawner \n", | |
| "20 jupyterhub/nbgitpuller-downloader-dropbox \n", | |
| "21 jupyterhub/nbgitpuller-downloader-generic-web \n", | |
| "22 jupyterhub/nbgitpuller-downloader-googledrive \n", | |
| "23 jupyterhub/nullauthenticator \n", | |
| "24 jupyterhub/pytest-jupyterhub \n", | |
| "25 jupyterhub/simpervisor \n", | |
| "26 jupyterhub/sudospawner \n", | |
| "27 jupyterhub/wrapspawner \n", | |
| "28 jupyterhub/dockerspawner \n", | |
| "29 jupyterhub/jupyter-remote-desktop-proxy \n", | |
| "30 jupyterhub/jupyter-rsession-proxy \n", | |
| "31 jupyterhealth/jupyter-smart-on-fhir \n", | |
| "32 jupyterhealth/jupyterhealth-client \n", | |
| "33 jupyterhub/nbgitpuller \n", | |
| "34 jupyterhub/oauthenticator \n", | |
| "35 jupyterhub/autodoc-traits \n", | |
| "36 jupyterhub/escapism \n", | |
| "37 jupyterhub/pamela \n", | |
| "\n", | |
| " package jupyter_org trusted_publisher \n", | |
| "0 batchspawner False False \n", | |
| "1 binderhub False False \n", | |
| "2 chartpress False False \n", | |
| "3 docker-image-cleaner False False \n", | |
| "4 gh-scoped-creds False False \n", | |
| "5 jupyter-repo2docker False False \n", | |
| "6 jupyter-server-proxy False False \n", | |
| "7 jupyterhub False False \n", | |
| "8 jupyterhub-firstuseauthenticator False False \n", | |
| "9 jupyterhub-idle-culler False False \n", | |
| "10 jupyterhub-kerberosauthenticator False False \n", | |
| "11 jupyterhub-kubespawner False False \n", | |
| "12 jupyterhub-ldapauthenticator False False \n", | |
| "13 jupyterhub-ltiauthenticator False False \n", | |
| "14 jupyterhub-nativeauthenticator False False \n", | |
| "15 jupyterhub-sphinx-theme False False \n", | |
| "16 jupyterhub-systemdspawner False False \n", | |
| "17 jupyterhub-tmpauthenticator False False \n", | |
| "18 jupyterhub-traefik-proxy False False \n", | |
| "19 jupyterhub-yarnspawner False False \n", | |
| "20 nbgitpuller-downloader-dropbox False False \n", | |
| "21 nbgitpuller-downloader-generic-web False False \n", | |
| "22 nbgitpuller-downloader-googledrive False False \n", | |
| "23 nullauthenticator False False \n", | |
| "24 pytest-jupyterhub False False \n", | |
| "25 simpervisor False False \n", | |
| "26 sudospawner False False \n", | |
| "27 wrapspawner False False \n", | |
| "28 dockerspawner False True \n", | |
| "29 jupyter-remote-desktop-proxy False True \n", | |
| "30 jupyter-rsession-proxy False True \n", | |
| "31 jupyter-smart-on-fhir False True \n", | |
| "32 jupyterhealth-client False True \n", | |
| "33 nbgitpuller False True \n", | |
| "34 oauthenticator False True \n", | |
| "35 autodoc-traits True False \n", | |
| "36 escapism True False \n", | |
| "37 pamela True False " | |
| ] | |
| }, | |
| "execution_count": 10, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "packages[[\"name\", \"package\", \"jupyter_org\", \"trusted_publisher\"]].sort_values(\n", | |
| " [\"jupyter_org\", \"trusted_publisher\", \"package\"]\n", | |
| ").reset_index(drop=True)" | |
| ] | |
| } | |
| ], | |
| "metadata": { | |
| "kernelspec": { | |
| "display_name": "Python 3 (ipykernel)", | |
| "language": "python", | |
| "name": "python3" | |
| }, | |
| "language_info": { | |
| "codemirror_mode": { | |
| "name": "ipython", | |
| "version": 3 | |
| }, | |
| "file_extension": ".py", | |
| "mimetype": "text/x-python", | |
| "name": "python", | |
| "nbconvert_exporter": "python", | |
| "pygments_lexer": "ipython3", | |
| "version": "3.11.10" | |
| }, | |
| "widgets": { | |
| "application/vnd.jupyter.widget-state+json": { | |
| "state": {}, | |
| "version_major": 2, | |
| "version_minor": 0 | |
| } | |
| } | |
| }, | |
| "nbformat": 4, | |
| "nbformat_minor": 5 | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment