Skip to content

Commit

Permalink
Revert "Use GraphQL to generate release content (#3848)"
Browse files Browse the repository at this point in the history 
This reverts commit b1b218a.
  • Loading branch information
@ludeeus
ludeeus committed Jul 4, 2024
1 parent b1b218a commit 17ccc3d
Show file tree
Hide file tree
Showing 11 changed files with 56 additions and 276 deletions.
29 changes: 9 additions & 20 deletions custom_components/hacs/repositories/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,6 @@
("last_version", None),
("manifest_name", None),
("open_issues", 0),
("prerelease", None),
("stargazers_count", 0),
("topics", []),
)
Expand Down Expand Up @@ -166,7 +165,6 @@ class RepositoryData:
manifest_name: str = None
new: bool = True
open_issues: int = 0
prerelease: str = None
published_tags: list[str] = []
releases: bool = False
selected_tag: str = None
Expand Down Expand Up @@ -571,11 +569,9 @@ async def download_zip_files(self, validate: Validate) -> None:
),
validate,
)
# lgtm [py/catch-base-exception] pylint: disable=broad-except
except BaseException:
except BaseException: # lgtm [py/catch-base-exception] pylint: disable=broad-except
validate.errors.append(
f"Download of {
self.repository_manifest.filename} was not completed"
f"Download of {self.repository_manifest.filename} was not completed"
)

async def async_download_zip_file(
Expand Down Expand Up @@ -614,8 +610,7 @@ def cleanup_temp_dir():
return

validate.errors.append(f"[{content['name']}] was not downloaded")
# lgtm [py/catch-base-exception] pylint: disable=broad-except
except BaseException:
except BaseException: # lgtm [py/catch-base-exception] pylint: disable=broad-except
validate.errors.append("Download was not completed")

async def download_content(self, version: string | None = None) -> None:
Expand Down Expand Up @@ -724,8 +719,7 @@ async def async_get_hacs_json(self, ref: str = None) -> dict[str, Any] | None:
)
if response:
return json_loads(decode_content(response.data.content))
# lgtm [py/catch-base-exception] pylint: disable=broad-except
except BaseException:
except BaseException: # lgtm [py/catch-base-exception] pylint: disable=broad-except
pass

async def async_get_info_file_contents(self, *, version: str | None = None, **kwargs) -> str:
Expand Down Expand Up @@ -826,8 +820,7 @@ async def remove_local_directory(self) -> None:
)

except (
# lgtm [py/catch-base-exception] pylint: disable=broad-except
BaseException
BaseException # lgtm [py/catch-base-exception] pylint: disable=broad-except
) as exception:
self.logger.debug("%s Removing %s failed with %s", self.string, local_path, exception)
return False
Expand Down Expand Up @@ -952,8 +945,7 @@ async def async_install_repository(self, *, version: str | None = None, **_) ->
):
persistent_directory = Backup(
hacs=self.hacs,
local_path=f"{
self.content.path.local}/{self.repository_manifest.persistent_directory}",
local_path=f"{self.content.path.local}/{self.repository_manifest.persistent_directory}",
backup_path=tempfile.gettempdir() + "/hacs_persistent_directory/",
)
await self.hacs.hass.async_add_executor_job(persistent_directory.create)
Expand Down Expand Up @@ -1280,8 +1272,7 @@ async def dowload_repository_content(self, content: FileInformation) -> None:
self.validate.errors.append(f"[{content.name}] was not downloaded.")

except (
# lgtm [py/catch-base-exception] pylint: disable=broad-except
BaseException
BaseException # lgtm [py/catch-base-exception] pylint: disable=broad-except
) as exception:
self.validate.errors.append(f"Download was not completed [{exception}]")

Expand Down Expand Up @@ -1341,8 +1332,7 @@ async def get_documentation(
return None

result = await self.hacs.async_download_file(
f"https://raw.githubusercontent.com/{
self.data.full_name}/{target_version}/{filename}",
f"https://raw.githubusercontent.com/{self.data.full_name}/{target_version}/{filename}",
nolog=True,
)

Expand All @@ -1359,8 +1349,7 @@ async def get_hacs_json(self, *, version: str, **kwargs) -> HacsManifest | None:
self.logger.debug("%s Getting hacs.json for version=%s", self.string, version)
try:
result = await self.hacs.async_download_file(
f"https://raw.githubusercontent.com/{
self.data.full_name}/{version}/hacs.json",
f"https://raw.githubusercontent.com/{self.data.full_name}/{version}/hacs.json",
nolog=True,
)
if result is None:
Expand Down
5 changes: 0 additions & 5 deletions custom_components/hacs/utils/data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@
("last_version", None),
("manifest_name", None),
("open_issues", 0),
("prerelease", None),
("published_tags", []),
("releases", False),
("selected_tag", None),
Expand Down Expand Up @@ -290,7 +289,6 @@ def async_restore_repository(self, entry: str, repository_data: dict[str, Any]):
repository.data.selected_tag = repository_data.get("selected_tag")
repository.data.show_beta = repository_data.get("show_beta", False)
repository.data.last_version = repository_data.get("last_version")
repository.data.prerelease = repository_data.get("prerelease")
repository.data.last_commit = repository_data.get("last_commit")
repository.data.installed_version = repository_data.get("version_installed")
repository.data.installed_commit = repository_data.get("installed_commit")
Expand All @@ -303,9 +301,6 @@ def async_restore_repository(self, entry: str, repository_data: dict[str, Any]):
repository_data.get("manifest") or repository_data.get("repository_manifest") or {}
)

if repository.data.prerelease == repository.data.last_version:
repository.data.prerelease = None

if repository.localpath is not None and is_safe(self.hacs, repository.localpath):
# Set local path
repository.content.path.local = repository.localpath
Expand Down
1 change: 0 additions & 1 deletion custom_components/hacs/utils/validate.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,6 @@ def validate_version(data: Any) -> Any:
vol.Required("last_fetched"): vol.Any(int, float),
vol.Required("last_updated"): str,
vol.Optional("last_version"): str,
vol.Optional("prerelease"): str,
vol.Required("manifest"): {
vol.Optional("country"): vol.Any([str], False),
vol.Optional("name"): str,
Expand Down
122 changes: 42 additions & 80 deletions scripts/data/generate_category_data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,34 +57,6 @@

OUTPUT_DIR = os.path.join(os.getcwd(), "outputdata")

GQL_RELEASES = """
query ($owner: String!, $repo: String!) {
repository(owner: $owner, name: $repo) {
latestRelease {
tagName
releaseAssets(first: 5) {
nodes {
name
downloadCount
}
}
}
releases(last: 1, orderBy: {field: CREATED_AT, direction: ASC}) {
nodes {
tagName
isPrerelease
releaseAssets(first: 5) {
nodes {
name
downloadCount
}
}
}
}
}
}
"""


def jsonprint(data: any):
print(
Expand Down Expand Up @@ -158,7 +130,9 @@ def async_store_repository_data(self, repository: HacsRepository) -> dict:
"""Store the repository data."""
data = {"manifest": {}}
for key, default in HACS_MANIFEST_KEYS_TO_EXPORT:
if (value := getattr(repository.repository_manifest, key, default)) != default:
if (
value := getattr(repository.repository_manifest, key, default)
) != default:
data["manifest"][key] = value

for key, default in REPOSITORY_KEYS_TO_EXPORT:
Expand Down Expand Up @@ -192,7 +166,8 @@ def __init__(self, session: ClientSession, *, token: str | None = None):
self.core.config_path = None
self.configuration.token = token
self.data = AdjustedHacsData(hacs=self)
self.data_client = HacsDataClient(session=session, client_name="HACS/Generator")
self.data_client = HacsDataClient(
session=session, client_name="HACS/Generator")

self.github = GitHub(
token,
Expand Down Expand Up @@ -235,61 +210,41 @@ async def concurrent_update_repository(self, repository: HacsRepository) -> None
"%s Fetching repository releases",
repository.string,
)

repoowner, reponame = repository.data.full_name.split("/")
response = await self.githubapi.graphql(
query=GQL_RELEASES,
variables={"owner": repoowner, "repo": reponame},
response = await self.githubapi.generic(
endpoint=f"/repos/{repository.data.full_name}/releases/latest",
etag=repository.data.etag_releases,
)

if (data := response.data["data"]["repository"]) is not None and (
last_release_nodes := data.get("releases", {}).get("nodes", [])
):
response.data = (
GitHubReleaseModel(
response.data) if response.data else None
)
repository.data.etag_releases = response.etag
if (releases := response.data) is not None:
repository.data.releases = True

latest_release = data.get("latestRelease", {})
last_release = last_release_nodes[0]

repository.releases.objects = [releases]
repository.data.published_tags = [
x.tag_name for x in repository.releases.objects
]
if (
repository.data.prerelease
and repository.data.prerelease != last_release["tagName"]
) or (
repository.data.last_version
and repository.data.last_version != latest_release["tagName"]
):
next_version := next(iter(repository.data.published_tags), None)
) != repository.data.last_version:
repository.data.last_version = next_version
repository.data.etag_repository = None

release_assets = latest_release.get("releaseAssets", {}).get("nodes", [])
repository.data.downloads = (
release_assets[0]["downloadCount"] if release_assets else 0
)
repository.data.published_tags = [repository.data.last_version]
repository.releases.objects = [
GitHubReleaseModel(
{
"tag_name": repository.data.last_version,
"assets": [
{
"name": a["name"],
"download_count": a["downloadCount"],
}
for a in release_assets
],
}
)
]

repository.data.prerelease = last_release.get("tagName")

if repository.data.prerelease == repository.data.last_version:
repository.data.prerelease = None

except GitHubNotModifiedException:
repository.data.releases = True
repository.logger.info(
"%s Release data is up to date",
repository.string,
)
except GitHubNotFoundException:
repository.data.releases = False
repository.logger.info("%s No releases found", repository.string)
repository.logger.info(
"%s No releases found", repository.string)
except GitHubException as exception:
repository.data.releases = False
repository.logger.warning("%s %s", repository.string, exception)
repository.logger.warning(
"%s %s", repository.string, exception)

await repository.common_update(
force=repository.data.etag_repository is None,
Expand Down Expand Up @@ -375,7 +330,8 @@ async def get_category_repositories(
continue
repository = self.repositories.get_by_full_name(repo)
if repository is not None:
self.queue.add(self.concurrent_update_repository(repository=repository))
self.queue.add(self.concurrent_update_repository(
repository=repository))
continue

self.queue.add(
Expand Down Expand Up @@ -453,7 +409,8 @@ async def async_github_get_hacs_default_file(self, filename: str) -> list:
async def generate_category_data(category: str, repository_name: str = None):
"""Generate data."""
async with ClientSession() as session:
hacs = AdjustedHacs(session=session, token=os.getenv("DATA_GENERATOR_TOKEN"))
hacs = AdjustedHacs(
session=session, token=os.getenv("DATA_GENERATOR_TOKEN"))
os.makedirs(os.path.join(OUTPUT_DIR, category), exist_ok=True)
os.makedirs(os.path.join(OUTPUT_DIR, "diff"), exist_ok=True)
force = os.environ.get("FORCE_REPOSITORY_UPDATE") == "True"
Expand Down Expand Up @@ -493,7 +450,11 @@ async def generate_category_data(category: str, repository_name: str = None):
)

did_raise = False
if not updated_data or len(updated_data) == 0 or not isinstance(updated_data, dict):
if (
not updated_data
or len(updated_data) == 0
or not isinstance(updated_data, dict)
):
print_error_and_exit(f"Updated data is empty", category)
did_raise = True

Expand All @@ -510,7 +471,8 @@ async def generate_category_data(category: str, repository_name: str = None):
print_error_and_exit(f"Invalid data: {errors}", category)

if did_raise:
print_error_and_exit("Validation did raise but did not exit!", category)
print_error_and_exit(
"Validation did raise but did not exit!", category)
sys.exit(1) # Fallback, should not be reached

with open(
Expand Down
17 changes: 0 additions & 17 deletions tests/output/proxy_calls.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1252,23 +1252,6 @@
"https://api.github.com/repos/hacs-test-org/theme-basic/releases": 1,
"https://data-v2.hacs.xyz/theme/data.json": 1
},
"tests/scripts/data/test_generate_category_data.py::test_generate_category_data_with_prior_content[category_test_data0]": {
"https://api.github.com/graphql": 1,
"https://api.github.com/rate_limit": 1,
"https://api.github.com/repos/hacs-test-org/integration-basic": 1,
"https://api.github.com/repos/hacs-test-org/integration-basic-custom": 1,
"https://api.github.com/repos/hacs-test-org/integration-basic-custom/contents/custom_components/example/manifest.json": 1,
"https://api.github.com/repos/hacs-test-org/integration-basic-custom/contents/hacs.json": 1,
"https://api.github.com/repos/hacs-test-org/integration-basic-custom/git/trees/1.0.0": 1,
"https://api.github.com/repos/hacs-test-org/integration-basic-custom/releases": 1,
"https://api.github.com/repos/hacs-test-org/integration-basic/git/trees/1.0.0": 1,
"https://api.github.com/repos/hacs/default/contents/integration": 1,
"https://api.github.com/repos/hacs/integration": 1,
"https://api.github.com/repos/hacs/integration/branches/main": 1,
"https://api.github.com/repos/hacs/integration/releases": 1,
"https://data-v2.hacs.xyz/integration/data.json": 1,
"https://data-v2.hacs.xyz/removed/repositories.json": 1
},
"tests/test_config_flow.py::test_flow_with_activation_failure": {
"https://github.com/login/device/code": 1,
"https://github.com/login/oauth/access_token": 2
Expand Down
Loading

0 comments on commit 17ccc3d

Please sign in to comment.