Coverage for webapp/topics/views.py: 32%

1import json 

2from os import getenv 

3 

4from canonicalwebteam.discourse import DocParser 

5from canonicalwebteam.discourse.exceptions import ( 

6 PathNotFoundError, 

7 RedirectFoundError, 

8) 

9from flask import Blueprint, abort, jsonify, render_template, request, redirect 

10from redis_cache.cache_utility import redis_cache 

11from webapp.helpers import discourse_api 

12from jinja2 import Template 

13from bs4 import BeautifulSoup 

14from urllib.parse import urlparse 

15 

16from webapp.config import CATEGORIES 

17from webapp.observability.utils import trace_function 

18 

19DISCOURSE_API_KEY = getenv("DISCOURSE_API_KEY") 

20DISCOURSE_API_USERNAME = getenv("DISCOURSE_API_USERNAME") 

21ALLOWED_HOST = "charmhub.io" 

22 

23topics = Blueprint( 

24 "topics", __name__, template_folder="/templates", static_folder="/static" 

25) 

26 

27with open("webapp/topics/topics.json") as f: 

28 topic_list = json.load(f) 

29 

30 

31class TopicParser(DocParser): 

32 @trace_function 

33 def parse_topic(self, topic, docs_version=""): 

34 result = super().parse_topic(topic, docs_version) 

35 

36 soup = BeautifulSoup(result["body_html"], features="html.parser") 

37 self._parse_packages(soup) 

38 result["body_html"] = str(soup) 

39 return result 

40 

41 @trace_function 

42 def _parse_packages(self, soup): 

43 """ 

44 Get a list of packages from all the 

45 packages tables in a topic 

46 

47 Example: 

48 | Charms | 

49 | -- | 

50 | https://charmhub.io/hello-kubecon | 

51 """ 

52 package_tables = [] 

53 

54 tables = soup.select("table:has(th:-soup-contains('Charms'))") 

55 

56 for table in tables: 

57 table_rows = table.select("tr:has(td)") 

58 

59 if table_rows: 

60 packages_set = {"soup_table": table, "packages": []} 

61 

62 # Get all packages URLs in this table 

63 for row in table_rows: 

64 navlink_href = row.find("a", href=True) 

65 

66 if navlink_href: 

67 navlink_href = navlink_href.get("href") 

68 parsed_url = urlparse(navlink_href) 

69 

70 if parsed_url.netloc != ALLOWED_HOST: 

71 self.warnings.append("Invalid tutorial URL") 

72 continue 

73 

74 # To avoid iframe issues with local development, demos 

75 navlink_href = navlink_href.replace( 

76 "https://charmhub.io/", request.url_root 

77 ) 

78 

79 packages_set["packages"].append(navlink_href) 

80 

81 package_tables.append(packages_set) 

82 

83 if package_tables: 

84 # Remplace tables with cards 

85 self._replace_packages(package_tables) 

86 

87 @trace_function 

88 def _replace_packages(self, package_tables): 

89 """ 

90 Replace charm tables to cards 

91 """ 

92 card_template = Template( 

93 ( 

94 '<div class="row">' 

95 "{% for package in packages %}" 

96 '<div class="col-small-5 col-medium-3 col-3">' 

97 '<iframe src="{{ package }}/embedded?store_design=true" ' 

98 'frameborder="0" width="100%" height="266px" ' 

99 'style="border: 0"></iframe>' 

100 "</div>" 

101 "{% endfor %}" 

102 "</div>" 

103 ) 

104 ) 

105 

106 for table in package_tables: 

107 card = card_template.render( 

108 packages=table["packages"], 

109 ) 

110 table["soup_table"].replace_with( 

111 BeautifulSoup(card, features="html.parser") 

112 ) 

113 

114 

115@trace_function 

116@topics.route("/topics.json") 

117def topics_json(): 

118 query = request.args.get("q", default=None, type=str) 

119 q = None if query in (None, "", "null") else query 

120 key = ("topics-json", {"q": q}) 

121 results = redis_cache.get(key, expected_type=list) 

122 if results: 

123 return jsonify( 

124 { 

125 "topics": results, 

126 "q": query, 

127 "size": len(results), 

128 } 

129 ) 

130 else: 

131 if query: 

132 query = query.lower() 

133 matched = [] 

134 unmatched = [] 

135 

136 for t in topic_list: 

137 if query in t["name"].lower() or query in t["categories"]: 

138 matched.append(t) 

139 else: 

140 unmatched.append(t) 

141 results = matched + unmatched 

142 else: 

143 results = topic_list 

144 redis_cache.set(key, results, ttl=43200) 

145 

146 return jsonify( 

147 { 

148 "topics": results, 

149 "q": query, 

150 "size": len(results), 

151 } 

152 ) 

153 

154 

155@trace_function 

156@topics.route("/topics") 

157def all_topics(): 

158 context = {} 

159 context["topics"] = topic_list 

160 context["categories"] = CATEGORIES 

161 return render_template("topics/index.html", **context) 

162 

163 

164@trace_function 

165@topics.route("/topics/<string:topic_slug>") 

166@topics.route("/topics/<string:topic_slug>/<path:path>") 

167def topic_page(topic_slug, path=None): 

168 key = ("topic-page", {"topic_slug": topic_slug, "path": path}) 

169 cached_page = redis_cache.get(key, expected_type=dict) 

170 if cached_page: 

171 return render_template("topics/document.html", **cached_page) 

172 topic = next((t for t in topic_list if t["slug"] == topic_slug), None) 

173 

174 if not topic: 

175 return abort(404) 

176 

177 topic_id = topic["topic_id"] 

178 docs_url_prefix = f"/topics/{topic_slug}" 

179 

180 docs = TopicParser( 

181 api=discourse_api, 

182 index_topic_id=topic_id, 

183 url_prefix=docs_url_prefix, 

184 tutorials_index_topic_id=2628, 

185 tutorials_url_prefix="https://juju.is/tutorials", 

186 ) 

187 docs.parse() 

188 

189 if path: 

190 try: 

191 topic_id = docs.resolve_path(path)[0] 

192 except PathNotFoundError: 

193 abort(404) 

194 except RedirectFoundError as path_redirect: 

195 return redirect(path_redirect.target_url) 

196 

197 topic = docs.api.get_topic(topic_id) 

198 else: 

199 topic = docs.index_topic 

200 

201 document = docs.parse_topic(topic) 

202 context = { 

203 "navigation": docs.navigation, 

204 "forum_url": docs.api.base_url, 

205 "document": document, 

206 } 

207 redis_cache.set(key, context, ttl=3600) 

208 

209 return render_template("topics/document.html", **context)