Coverage for webapp/publisher/snaps/build_views.py: 48%
250 statements
« prev ^ index » next coverage.py v7.15.4, created at 2026-08-20 22:09 +0000
1# Standard library
2import os
3import re
4from hashlib import md5
5from urllib.parse import urljoin
7# Packages
8import flask
9from canonicalwebteam.store_api.dashboard import Dashboard
11from requests.exceptions import HTTPError
13# Local
14from webapp.api.exceptions import ApiConnectionError, ApiTimeoutError
15from webapp.helpers import api_publisher_session, launchpad
16from webapp.api.github import GitHub, InvalidYAML
17from webapp.decorators import login_required
18from webapp.extensions import csrf
19from webapp.publisher.snaps.builds import map_build_and_upload_states
20from werkzeug.exceptions import Unauthorized
22GITHUB_SNAPCRAFT_USER_TOKEN = os.getenv("GITHUB_SNAPCRAFT_USER_TOKEN")
23GITHUB_WEBHOOK_HOST_URL = os.getenv("GITHUB_WEBHOOK_HOST_URL")
26def extract_github_repository(git_repository_url):
27 """
28 Extract owner/repo from a GitHub repository URL.
30 Args:
31 git_repository_url (str): The full GitHub repository URL
33 Returns:
34 str or None: The owner/repo part of the URL, or None if not a
35 valid GitHub URL
36 """
37 if not git_repository_url:
38 return None
40 match = re.search(
41 r"github\.com/(?P<repo>.+/.+?)(?:\.git)?/?$", git_repository_url
42 )
43 if match:
44 return match.groupdict()["repo"]
45 return None
48BUILDS_PER_PAGE = 15
49BUILD_LOG_REQUEST_TIMEOUT = (30, 60)
50dashboard = Dashboard(api_publisher_session)
53def get_builds(lp_snap, selection):
54 builds = launchpad.get_snap_builds(lp_snap["store_name"])
56 total_builds = len(builds)
58 builds = builds[selection]
60 snap_builds = []
61 builders_status = None
63 # Extract GitHub repository info for commit links
64 github_repository = extract_github_repository(
65 lp_snap.get("git_repository_url")
66 )
68 for build in builds:
69 status = map_build_and_upload_states(
70 build["buildstate"], build["store_upload_status"]
71 )
73 snap_build = {
74 "id": build["self_link"].split("/")[-1],
75 "arch_tag": build["arch_tag"],
76 "datebuilt": build["datebuilt"],
77 "duration": build["duration"],
78 "logs": build["build_log_url"],
79 "revision_id": build["revision_id"],
80 "status": status,
81 "title": build["title"],
82 "queue_time": None,
83 "github_repository": github_repository,
84 }
86 if build["buildstate"] == "Needs building":
87 if not builders_status:
88 builders_status = launchpad.get_builders_status()
90 snap_build["queue_time"] = builders_status[build["arch_tag"]][
91 "estimated_duration"
92 ]
94 snap_builds.append(snap_build)
96 return {
97 "total_builds": total_builds,
98 "snap_builds": snap_builds,
99 }
102@login_required
103def get_snap_builds_page(snap_name):
104 # If this fails, the page will 404
105 dashboard.get_snap_info(flask.session, snap_name)
106 return flask.render_template("store/publisher.html", snap_name=snap_name)
109@login_required
110def get_snap_builds(snap_name):
111 res = {"message": "", "success": True}
112 data = {"snap_builds": [], "total_builds": 0}
114 details = dashboard.get_snap_info(flask.session, snap_name)
115 start = flask.request.args.get("start", 0, type=int)
116 size = flask.request.args.get("size", 15, type=int)
117 build_slice = slice(start, size)
119 # Get built snap in launchpad with this store name
120 lp_snap = launchpad.get_snap_by_store_name(details["snap_name"])
122 if lp_snap:
123 data.update(get_builds(lp_snap, build_slice))
125 res["data"] = data
127 return flask.jsonify(res)
130@login_required
131def get_snap_build(snap_name, build_id):
132 details = dashboard.get_snap_info(flask.session, snap_name)
134 context = {
135 "snap_id": details["snap_id"],
136 "snap_name": details["snap_name"],
137 "snap_title": details["title"],
138 "snap_build": {},
139 }
141 # Get build by snap name and build_id
142 lp_build = launchpad.get_snap_build(details["snap_name"], build_id)
144 if lp_build:
145 # Get snap info to extract GitHub repository
146 lp_snap = launchpad.get_snap_by_store_name(details["snap_name"])
147 github_repository = None
148 if lp_snap:
149 github_repository = extract_github_repository(
150 lp_snap.get("git_repository_url")
151 )
153 status = map_build_and_upload_states(
154 lp_build["buildstate"], lp_build["store_upload_status"]
155 )
156 context["snap_build"] = {
157 "id": lp_build["self_link"].split("/")[-1],
158 "arch_tag": lp_build["arch_tag"],
159 "datebuilt": lp_build["datebuilt"],
160 "duration": lp_build["duration"],
161 "logs": lp_build["build_log_url"],
162 "revision_id": lp_build["revision_id"],
163 "status": status,
164 "title": lp_build["title"],
165 "github_repository": github_repository,
166 }
168 return flask.jsonify({"data": context, "success": True})
171@login_required
172def get_snap_build_logs(snap_name, build_id):
173 details = dashboard.get_snap_info(flask.session, snap_name)
174 lp_build = launchpad.get_snap_build(details["snap_name"], build_id)
176 if not lp_build:
177 return (
178 flask.jsonify(
179 {
180 "error": {
181 "message": "The requested build could not be found."
182 },
183 "success": False,
184 }
185 ),
186 404,
187 )
189 if not lp_build["build_log_url"]:
190 return (
191 flask.jsonify(
192 {
193 "error": {"message": "The requested build has no log."},
194 "success": False,
195 }
196 ),
197 404,
198 )
200 log_url = lp_build["build_log_url"]
201 response = None
203 try:
204 response = api_publisher_session.get(
205 log_url,
206 headers={"Accept": "text/plain"},
207 stream=True,
208 timeout=BUILD_LOG_REQUEST_TIMEOUT,
209 allow_redirects=False,
210 )
211 response.raise_for_status()
213 if response.is_redirect:
214 redirect_url = response.headers.get("Location")
215 response.close()
217 if not redirect_url:
218 raise HTTPError(response=response)
220 response = api_publisher_session.get(
221 urljoin(log_url, redirect_url),
222 headers={"Accept": "text/plain"},
223 stream=True,
224 timeout=BUILD_LOG_REQUEST_TIMEOUT,
225 )
226 response.raise_for_status()
227 except (ApiConnectionError, ApiTimeoutError, HTTPError):
228 if response:
229 response.close()
231 return (
232 flask.jsonify(
233 {
234 "error": {
235 "message": "The requested build log could not be "
236 "fetched."
237 },
238 "success": False,
239 }
240 ),
241 502,
242 )
244 def generate_log_chunks():
245 try:
246 for chunk in response.iter_content(
247 chunk_size=8192, decode_unicode=True
248 ):
249 if chunk:
250 yield chunk
251 finally:
252 response.close()
254 return flask.Response(
255 flask.stream_with_context(generate_log_chunks()),
256 mimetype="text/plain",
257 )
260def validate_repo(github_token, snap_name, gh_owner, gh_repo):
261 github = GitHub(github_token)
262 result = {"success": True}
263 yaml_location = github.get_snapcraft_yaml_location(gh_owner, gh_repo)
265 # The snapcraft.yaml is not present
266 if not yaml_location:
267 result["success"] = False
268 result["error"] = {
269 "type": "MISSING_YAML_FILE",
270 "message": (
271 "Missing snapcraft.yaml: this repo needs a snapcraft.yaml "
272 "file, so that Snapcraft can make it buildable, installable "
273 "and runnable."
274 ),
275 }
276 # The property name inside the yaml file doesn't match the snap
277 else:
278 try:
279 gh_snap_name = github.get_snapcraft_yaml_data(
280 gh_owner, gh_repo
281 ).get("name")
283 if gh_snap_name != snap_name:
284 result["success"] = False
285 result["error"] = {
286 "type": "SNAP_NAME_DOES_NOT_MATCH",
287 "message": (
288 "Name mismatch: the snapcraft.yaml uses the snap "
289 f'name "{gh_snap_name}", but you\'ve registered'
290 f' the name "{snap_name}". Update your '
291 "snapcraft.yaml to continue."
292 ),
293 "yaml_location": yaml_location,
294 "gh_snap_name": gh_snap_name,
295 }
296 except InvalidYAML:
297 result["success"] = False
298 result["error"] = {
299 "type": "INVALID_YAML_FILE",
300 "message": (
301 "Invalid snapcraft.yaml: there was an issue parsing the "
302 f"snapcraft.yaml for {snap_name}."
303 ),
304 }
306 return result
309@login_required
310def post_snap_builds(snap_name):
311 details = dashboard.get_snap_info(flask.session, snap_name)
313 # Don't allow changes from Admins that are no contributors
314 account_snaps = dashboard.get_account_snaps(flask.session)
316 if snap_name not in account_snaps:
317 flask.flash(
318 "You do not have permissions to modify this Snap", "negative"
319 )
320 return flask.redirect(
321 flask.url_for(".get_snap_builds_page", snap_name=snap_name)
322 )
324 redirect_url = flask.url_for(".get_snap_builds_page", snap_name=snap_name)
326 # Get built snap in launchpad with this store name
327 github = GitHub(flask.session.get("github_auth_secret"))
328 owner, repo = flask.request.form.get("github_repository").split("/")
330 if not github.check_permissions_over_repo(owner, repo):
331 flask.flash(
332 "The repository doesn't exist or you don't have"
333 " enough permissions",
334 "negative",
335 )
336 return flask.redirect(redirect_url)
338 repo_validation = validate_repo(
339 flask.session.get("github_auth_secret"), snap_name, owner, repo
340 )
342 if not repo_validation["success"]:
343 flask.flash(repo_validation["error"]["message"], "negative")
344 return flask.redirect(redirect_url)
346 lp_snap = launchpad.get_snap_by_store_name(details["snap_name"])
347 git_url = f"https://github.com/{owner}/{repo}"
349 # Root macaroon obtained from the store, to authorize builds to be
350 # uploaded to the store from Launchpad. This macaroon carries an SSO
351 # third-party caveat that only login.ubuntu.com can discharge, and that
352 # discharge is specific to *this* macaroon's caveat_id (it cannot be
353 # satisfied by reusing a discharge obtained for a different macaroon,
354 # e.g. the one from the user's original login). Without a matching
355 # discharge, Launchpad will accept the authorization call but can never
356 # actually use it to upload builds, and builds will be stuck as
357 # "Unscheduled" ("Won't release") forever with no visible error.
358 upload_macaroon = dashboard.get_package_upload_macaroon(
359 session=flask.session, snap_name=snap_name, channels=["edge"]
360 )["macaroon"]
362 if not lp_snap:
363 lp_snap_name = md5(git_url.encode("UTF-8")).hexdigest()
365 try:
366 repo_exist = launchpad.get_snap(lp_snap_name)
367 except HTTPError as e:
368 if e.response.status_code == 404:
369 repo_exist = False
370 else:
371 raise e
373 if repo_exist:
374 flask.flash(
375 "The specified repository is being used by another snap:"
376 f" {repo_exist['store_name']}",
377 "negative",
378 )
379 return flask.redirect(redirect_url)
381 pending_action = "link"
383 elif lp_snap["git_repository_url"] != git_url:
384 # In the future, create a new record, delete the old one
385 raise AttributeError(
386 f"Snap {snap_name} already has a build repository associated"
387 )
388 else:
389 pending_action = "repair"
391 # We can't complete the store authorization yet: we still need a
392 # discharge macaroon that specifically discharges `upload_macaroon`'s
393 # SSO caveat, which requires a redirect round-trip through
394 # login.ubuntu.com (see webapp/login/views.py:authorize_snap_build).
395 # Stash everything needed to finish the job once we're back, and kick
396 # off that round-trip.
397 flask.session["pending_snap_authorization"] = {
398 "action": pending_action,
399 "snap_name": snap_name,
400 "git_url": git_url,
401 "owner": owner,
402 "repo": repo,
403 "lp_snap_name": lp_snap["name"] if lp_snap else None,
404 "root_macaroon": upload_macaroon,
405 "redirect_url": redirect_url,
406 }
408 # This endpoint is called via `fetch()` from the React frontend, so we
409 # can't just return an HTTP redirect here: the browser needs to
410 # actually navigate away to complete the login.ubuntu.com round-trip,
411 # which a fetch() call can't do on the page's behalf. Signal the
412 # frontend to do that navigation itself.
413 return flask.jsonify(
414 {
415 "success": False,
416 "authorization_required": True,
417 "redirect_url": flask.url_for("login.authorize_snap_build"),
418 }
419 )
422def complete_pending_snap_authorization(pending, discharge_macaroon):
423 """
424 Finish linking/repairing a snap's Launchpad build authorization,
425 once a discharge macaroon for the pending upload macaroon's SSO
426 caveat has been obtained (see webapp/login/views.py).
427 """
428 snap_name = pending["snap_name"]
429 git_url = pending["git_url"]
430 redirect_url = pending["redirect_url"]
432 if pending["action"] == "link":
433 launchpad.create_snap(
434 snap_name,
435 git_url,
436 pending["root_macaroon"],
437 discharge_macaroon=discharge_macaroon,
438 )
440 flask.flash(
441 "The GitHub repository was linked successfully.", "positive"
442 )
444 owner = pending["owner"]
445 repo = pending["repo"]
446 github = GitHub(flask.session.get("github_auth_secret"))
448 # Create webhook in the repo, it should also trigger the first build
449 github_hook_url = (
450 f"{GITHUB_WEBHOOK_HOST_URL}api/{snap_name}/webhook/notify"
451 )
452 try:
453 hook = github.get_hook_by_url(owner, repo, github_hook_url)
455 # We create the webhook if doesn't exist already in this repo
456 if not hook:
457 github.create_hook(owner, repo, github_hook_url)
458 except HTTPError:
459 flask.flash(
460 "The GitHub Webhook could not be created. "
461 "Please trigger a new build manually.",
462 "caution",
463 )
464 else:
465 # The repo is already linked to this snap: re-run the store
466 # authorization handshake. This lets users self-repair snaps
467 # whose authorization silently failed (e.g. it was completed
468 # without a matching discharge macaroon), without having to
469 # unlink and relink the repository.
470 launchpad.complete_snap_authorization(
471 pending["lp_snap_name"],
472 pending["root_macaroon"],
473 discharge_macaroon=discharge_macaroon,
474 )
475 flask.flash(
476 "The GitHub repository authorization was refreshed.",
477 "positive",
478 )
480 return flask.redirect(redirect_url)
482 return flask.redirect(redirect_url)
485@login_required
486def check_build_request(snap_name, build_id):
487 # Don't allow builds from no contributors
488 account_snaps = dashboard.get_account_snaps(flask.session)
490 if snap_name not in account_snaps:
491 return flask.jsonify(
492 {
493 "success": False,
494 "error": {
495 "type": "FORBIDDEN",
496 "message": "You are not allowed to request "
497 "builds for this snap",
498 },
499 }
500 )
502 try:
503 response = launchpad.get_snap_build_request(snap_name, build_id)
504 except HTTPError as e:
505 # Timeout or not found from Launchpad
506 if e.response.status_code in [408, 404]:
507 return flask.jsonify(
508 {
509 "success": False,
510 "error": {
511 "message": "An error happened building "
512 "this snap, please try again."
513 },
514 }
515 )
516 raise e
518 error_message = None
519 if response["error_message"]:
520 error_message = response["error_message"].split(" HEAD:")[0]
522 return flask.jsonify(
523 {
524 "success": True,
525 "status": response["status"],
526 "error": {"message": error_message},
527 }
528 )
531@csrf.exempt
532def post_github_webhook(snap_name=None, github_owner=None, github_repo=None):
533 payload = flask.request.json
534 repo_url = payload["repository"]["html_url"]
535 gh_owner = payload["repository"]["owner"]["login"]
536 gh_repo = payload["repository"]["name"]
537 gh_default_branch = payload["repository"]["default_branch"]
539 # The first payload after the webhook creation
540 # doesn't contain a "ref" key
541 if "ref" in payload:
542 gh_event_branch = payload["ref"][11:]
543 else:
544 gh_event_branch = gh_default_branch
546 # Check the push event is in the default branch
547 if gh_default_branch != gh_event_branch:
548 return ("The push event is not for the default branch", 200)
550 if snap_name:
551 lp_snap = launchpad.get_snap_by_store_name(snap_name)
552 else:
553 lp_snap = launchpad.get_snap(md5(repo_url.encode("UTF-8")).hexdigest())
555 if not lp_snap:
556 return ("This repository is not linked with any Snap", 403)
558 # Check that this is the repo for this snap
559 if lp_snap["git_repository_url"] != repo_url:
560 return ("The repository does not match the one used by this Snap", 403)
562 github = GitHub()
564 signature = flask.request.headers.get("X-Hub-Signature")
566 if not github.validate_webhook_signature(flask.request.data, signature):
567 if not github.validate_bsi_webhook_secret(
568 gh_owner, gh_repo, flask.request.data, signature
569 ):
570 return ("Invalid secret", 403)
572 validation = validate_repo(
573 GITHUB_SNAPCRAFT_USER_TOKEN, lp_snap["store_name"], gh_owner, gh_repo
574 )
576 if not validation["success"]:
577 return (validation["error"]["message"], 400)
579 if launchpad.is_snap_building(lp_snap["store_name"]):
580 launchpad.cancel_snap_builds(lp_snap["store_name"])
582 launchpad.build_snap(lp_snap["store_name"])
584 return ("", 204)
587@login_required
588def get_update_gh_webhooks(snap_name):
589 details = dashboard.get_snap_info(flask.session, snap_name)
591 lp_snap = launchpad.get_snap_by_store_name(details["snap_name"])
593 if not lp_snap:
594 flask.flash(
595 "This snap is not linked with a GitHub repository", "negative"
596 )
598 return flask.redirect(
599 flask.url_for(".get_settings", snap_name=snap_name)
600 )
602 github = GitHub(flask.session.get("github_auth_secret"))
604 try:
605 github.get_user()
606 except Unauthorized:
607 return flask.redirect(
608 flask.url_for("oauth.github_auth", back=flask.request.path)
609 )
611 gh_link = lp_snap["git_repository_url"][19:]
612 gh_owner, gh_repo = gh_link.split("/")
614 try:
615 # Remove old BSI webhook if present
616 old_url = (
617 f"https://build.snapcraft.io/{gh_owner}/{gh_repo}/webhook/notify"
618 )
619 old_hook = github.get_hook_by_url(gh_owner, gh_repo, old_url)
621 if old_hook:
622 github.remove_hook(
623 gh_owner,
624 gh_repo,
625 old_hook["id"],
626 )
628 # Remove current hook
629 github_hook_url = (
630 f"{GITHUB_WEBHOOK_HOST_URL}api/{snap_name}/webhook/notify"
631 )
632 snapcraft_hook = github.get_hook_by_url(
633 gh_owner, gh_repo, github_hook_url
634 )
636 if snapcraft_hook:
637 github.remove_hook(
638 gh_owner,
639 gh_repo,
640 snapcraft_hook["id"],
641 )
643 # Create webhook in the repo
644 github.create_hook(gh_owner, gh_repo, github_hook_url)
645 except HTTPError:
646 flask.flash(
647 "The GitHub Webhook could not be created. "
648 "Please try again or check your permissions over the repository.",
649 "caution",
650 )
651 else:
652 flask.flash("The webhook has been created successfully", "positive")
654 return flask.redirect(flask.url_for(".get_settings", snap_name=snap_name))