Coverage for webapp/api/launchpad_provenance.py: 96%

1import os 

2import re 

3from concurrent.futures import ThreadPoolExecutor 

4 

5from webapp.api.exceptions import ApiTimeoutError 

6from webapp.api.requests import Session 

7 

8MAX_CONCURRENT_SCANS = 10 

9 

10LAUNCHPAD_API_URL = os.getenv( 

11 "LAUNCHPAD_API_URL", "https://api.launchpad.net/devel/" 

12) 

13 

14 

15GITHUB_URL_RE = re.compile( 

16 r"^(?:https?://)?(?:www\.)?github\.com/" 

17 r"(?P<repo>[^/]+/[^/]+?)(?:\.git)?/?$" 

18) 

19 

20LAUNCHPAD_GIT_RE = re.compile( 

21 r"^https?://api\.launchpad\.net/[^/]+/" 

22 r"(?P<path>~[^/]+/(?:[^/]+/)*\+git/[^/]+?)/?$" 

23) 

24 

25 

26def failure_reason(exc): 

27 if isinstance(exc, ApiTimeoutError): 

28 return "launchpad_timeout" 

29 return "launchpad_error" 

30 

31 

32def extract_github_repository(git_repository_url): 

33 """Extract owner/repo from a GitHub repository URL, or None. 

34 

35 Anchored at both ends: the badge attests against this, so a URL merely 

36 containing "github.com/" must not match. 

37 """ 

38 if not git_repository_url: 

39 return None 

40 

41 match = GITHUB_URL_RE.match(git_repository_url) 

42 if match: 

43 return match.groupdict()["repo"] 

44 return None 

45 

46 

47def extract_launchpad_repository(git_repository_link): 

48 """Extract the git.launchpad.net path from a Launchpad link""" 

49 if not git_repository_link: 

50 return None 

51 

52 match = LAUNCHPAD_GIT_RE.match(git_repository_link) 

53 if match: 

54 return match.groupdict()["path"] 

55 return None 

56 

57 

58def build_commit_url(source, commit_sha): 

59 """Build a browsable commit URL for a recipe's source""" 

60 if source.get("github_repository"): 

61 return ( 

62 f"https://github.com/{source['github_repository']}" 

63 f"/commit/{commit_sha}" 

64 ) 

65 if source.get("launchpad_repository"): 

66 return ( 

67 f"https://git.launchpad.net/{source['launchpad_repository']}" 

68 f"/commit/?id={commit_sha}" 

69 ) 

70 return None 

71 

72 

73def recipe_source(recipe): 

74 """Resolve where a recipe's source is hosted.""" 

75 git_repository_url = recipe.get("git_repository_url") 

76 return { 

77 "git_repository_url": git_repository_url, 

78 "github_repository": extract_github_repository(git_repository_url), 

79 "launchpad_repository": extract_launchpad_repository( 

80 recipe.get("git_repository_link") 

81 ), 

82 } 

83 

84 

85class LaunchpadProvenance: 

86 """Read-only, anonymous client for Launchpad build provenance. 

87 

88 Unlike the authenticated ``canonicalwebteam.launchpad`` client, this one 

89 sends no OAuth credentials and does not filter recipes by owner, so it can 

90 read provenance for *any* public Launchpad recipe. It is used to link a 

91 store revision back to the public git commit it was built from. 

92 """ 

93 

94 def __init__(self, session=None, api_url=LAUNCHPAD_API_URL): 

95 self.api_url = api_url 

96 self._owns_session = session is None 

97 self.session = session or Session() 

98 self.session.headers["Accept"] = "application/json" 

99 

100 def _get(self, url, params=None, session=None): 

101 response = (session or self.session).get(url, params=params) 

102 response.raise_for_status() 

103 return response.json() 

104 

105 def _new_session(self): 

106 if not self._owns_session: 

107 return self.session 

108 session = Session() 

109 session.headers["Accept"] = "application/json" 

110 return session 

111 

112 def get_recipes(self, store_name, max_recipes): 

113 """Find the public Launchpad recipes for a store name, best first. 

114 

115 store_name is free text, so one name matches dozens of recipes 

116 (firefox matches 62), mostly personal ones that never upload. Ranked 

117 by upload capability, then recency; at most ``max_recipes``. 

118 """ 

119 data = self._get( 

120 f"{self.api_url}+snaps", 

121 params={ 

122 "ws.op": "findByStoreName", 

123 "store_name": f'"{store_name}"', 

124 }, 

125 ) 

126 

127 matches = [ 

128 entry 

129 for entry in data.get("entries", []) 

130 if entry.get("store_name") == store_name 

131 ] 

132 

133 # Two stable sorts: the second keeps the date order within each group. 

134 matches.sort( 

135 key=lambda e: e.get("date_last_modified") or "", reverse=True 

136 ) 

137 matches.sort(key=lambda e: not e.get("can_upload_to_store")) 

138 

139 return matches[:max_recipes] 

140 

141 def iter_builds(self, collection_link, max_pages, session=None): 

142 """Collect completed builds""" 

143 entries = [] 

144 url = collection_link 

145 pages = 0 

146 

147 while url and pages < max_pages: 

148 try: 

149 data = self._get(url, session=session) 

150 except Exception as exc: 

151 return entries, failure_reason(exc) 

152 entries.extend(data.get("entries", [])) 

153 url = data.get("next_collection_link") 

154 pages += 1 

155 

156 return entries, None 

157 

158 def _scan_recipe(self, recipe, max_pages): 

159 session = self._new_session() 

160 try: 

161 return self.iter_builds( 

162 recipe["completed_builds_collection_link"], 

163 max_pages, 

164 session=session, 

165 ) 

166 finally: 

167 if session is not self.session: 

168 session.close() 

169 

170 def _merge_builds(self, builds, source, revisions): 

171 """Fold one recipe's builds into the shared revision map. 

172 

173 Returns True if anything was added; earlier recipes win on conflict. 

174 """ 

175 added = False 

176 

177 for build in builds: 

178 if build.get("store_upload_status") != "Uploaded": 

179 continue 

180 

181 revision = build.get("store_upload_revision") 

182 commit_sha = build.get("revision_id") 

183 arch = build.get("arch_tag") 

184 

185 if not revision or not commit_sha or not arch: 

186 continue 

187 

188 revision_key = str(revision) 

189 # Newest first, so the first build seen for a revision+arch wins. 

190 arch_map = revisions.setdefault(revision_key, {}) 

191 if arch in arch_map: 

192 continue 

193 

194 commit_url = build_commit_url(source, commit_sha) 

195 

196 build_id = None 

197 build_url = None 

198 self_link = build.get("self_link") 

199 if self_link: 

200 build_id = self_link.rstrip("/").split("/")[-1] 

201 # API self_link -> human-facing web URL. 

202 build_url = self_link.replace( 

203 "api.launchpad.net/devel/", "launchpad.net/" 

204 ) 

205 

206 arch_map[arch] = { 

207 "commit_sha": commit_sha, 

208 "commit_url": commit_url, 

209 "build_id": build_id, 

210 "build_url": build_url, 

211 # Per entry: a merged map can span recipes with different 

212 # sources. 

213 "github_repository": source.get("github_repository"), 

214 "launchpad_repository": source.get("launchpad_repository"), 

215 } 

216 added = True 

217 

218 return added 

219 

220 def build_provenance_map(self, store_name, max_pages, max_recipes): 

221 """Return a provenance map joining store revisions to git commits. 

222 

223 Shape: 

224 { 

225 "github_repository": "owner/repo" | None, 

226 "launchpad_repository": "~owner/proj/+git/name" | None, 

227 "git_repository_url": "https://..." | None, 

228 "revisions": { 

229 "<store_revision>": { 

230 "<arch>": { 

231 "commit_sha": "...", 

232 "commit_url": "https://github.com/.../commit/..." 

233 | None, 

234 "build_id": "216436", 

235 }, 

236 }, 

237 }, 

238 } 

239 

240 Builds are merged across candidate recipes, since one store name can 

241 span several legitimate recipes each holding part of the history. 

242 Only uploaded builds with a ``revision_id`` are included; revision 

243 keys are strings so the map survives JSON round-trips. 

244 """ 

245 result = { 

246 "github_repository": None, 

247 "launchpad_repository": None, 

248 "git_repository_url": None, 

249 "revisions": {}, 

250 # Set when an upstream request failed, so callers can avoid 

251 # caching a transient failure as a negative answer. 

252 "failed": False, 

253 "reason": None, 

254 } 

255 

256 try: 

257 recipes = self.get_recipes(store_name, max_recipes) 

258 except Exception as exc: 

259 result["failed"] = True 

260 result["reason"] = failure_reason(exc) 

261 return result 

262 

263 if not recipes: 

264 return result 

265 

266 candidates = [ 

267 recipe 

268 for recipe in recipes 

269 if recipe.get("completed_builds_collection_link") 

270 ] 

271 if not candidates: 

272 return result 

273 

274 # Share the page budget across candidates: revisions being resolved 

275 # are recent and builds are newest first, so breadth beats depth. 

276 pages_each = max(1, max_pages // len(candidates)) 

277 

278 # receopes are fetched in parallel 

279 with ThreadPoolExecutor( 

280 max_workers=min(MAX_CONCURRENT_SCANS, len(candidates)) 

281 ) as executor: 

282 scans = list( 

283 executor.map( 

284 lambda recipe: self._scan_recipe(recipe, pages_each), 

285 candidates, 

286 ) 

287 ) 

288 

289 revisions = result["revisions"] 

290 source = None 

291 fallback = None 

292 

293 for recipe, (builds, reason) in zip(candidates, scans): 

294 recipe_src = recipe_source(recipe) 

295 if fallback is None: 

296 fallback = recipe_src 

297 

298 if ( 

299 self._merge_builds(builds, recipe_src, revisions) 

300 and source is None 

301 ): 

302 # The recipe that produced revisions, not the first sorted. 

303 source = recipe_src 

304 

305 if reason and not result["failed"]: 

306 result["failed"] = True 

307 result["reason"] = reason 

308 

309 if source is None: 

310 source = fallback 

311 if source: 

312 result["git_repository_url"] = source["git_repository_url"] 

313 result["github_repository"] = source["github_repository"] 

314 result["launchpad_repository"] = source["launchpad_repository"] 

315 

316 return result