Coverage for webapp/search/logic.py: 19%

1import requests 

2 

3from redis_cache.cache_utility import redis_cache 

4from webapp.config import SEARCH_FIELDS 

5from webapp.packages.logic import parse_package_for_card 

6from webapp.observability.utils import trace_function 

7from webapp.store_api import publisher_gateway 

8 

9DISCOURSE_URL = "https://discourse.charmhub.io" 

10DOCS_URL = "https://canonical-juju.readthedocs-hosted.com/" 

11 

12 

13@trace_function 

14def search_topics( 

15 query: str, 

16 page: int = 1, 

17 see_all: bool = False, 

18) -> list: 

19 """ 

20 Searches discourse for topics based on the query parameters. 

21 

22 Parameters: 

23 term (str): The search term used to find relevant topics. 

24 page (int): The page number of the search results to retrieve. 

25 category (str): The category to search from. 

26 see_all (bool, optional): If True, retrieves all available search 

27 results. If False (default), returns a limited number of results 

28 (5 posts and topics). 

29 

30 Returns: 

31 list: A list containing the a list of topics. 

32 

33 Note: 

34 This function makes use of a cache to store result for a fetched search 

35 terms, this helps in reducing redundant requests to the discourse API. 

36 """ 

37 key = ("search-topics", {"q": query, "pg": page}) 

38 cached_page = redis_cache.get(key, expected_type=list) 

39 

40 if not see_all: 

41 if cached_page: 

42 return cached_page 

43 else: 

44 resp = requests.get( 

45 f"{DISCOURSE_URL}/search.json?q={query}&page={page}" 

46 ) 

47 topics = resp.json().get("topics", []) 

48 for topic in topics: 

49 post = next( 

50 ( 

51 post 

52 for post in resp.json()["posts"] 

53 if post["topic_id"] == topic["id"] 

54 ), 

55 None, 

56 ) 

57 topic["post"] = post 

58 topics = [topic for topic in topics if topic["category_id"] != 22] 

59 redis_cache.set(key, topics, ttl=3600) 

60 return topics 

61 

62 # Note: this logic is currently slower than it should ordinarily 

63 # be because the discourse API currently has some limitations that 

64 # would probably be fixed in the near future. 

65 # The ones affecting this code are: 

66 # 1. The API does not return any indicator to show if there are more 

67 # pages to be fetched. 

68 # 2. The API does not support fetching multiple categories or 

69 # excluding a category from the search 

70 

71 result = [] 

72 more_pages = True 

73 

74 while more_pages: 

75 key = ("search-topics", {"q": query, "pg": page}) 

76 cached_page = redis_cache.get(key, expected_type=list) 

77 if cached_page: 

78 result.extend(cached_page) 

79 page += 1 

80 continue 

81 

82 resp = requests.get( 

83 f"{DISCOURSE_URL}/search.json?q={query}&page={page}" 

84 ) 

85 data = resp.json() 

86 topics = data.get("topics", []) 

87 

88 if topics: 

89 for topic in topics: 

90 post = next( 

91 ( 

92 post 

93 for post in data["posts"] 

94 if post["topic_id"] == topic["id"] 

95 ), 

96 None, 

97 ) 

98 topic["post"] = post 

99 redis_cache.set(key, topics, ttl=3600) 

100 result.extend(topics) 

101 page += 1 

102 key = ("search-topics", {"q": query, "pg": page}) 

103 

104 cached_next_topics = redis_cache.get(key, expected_type=list) 

105 if cached_next_topics: 

106 next_topics = cached_next_topics 

107 else: 

108 next_resp = requests.get( 

109 f"{DISCOURSE_URL}/search.json?q={query}&page={page}" 

110 ) 

111 next_topics = [ 

112 topic 

113 for topic in next_resp.json().get("topics", []) 

114 if topic["category_id"] != 22 

115 ] 

116 redis_cache.set(key, next_topics, ttl=3600) 

117 if not next_topics or next_topics[0]["id"] == topics[0]["id"]: 

118 more_pages = False 

119 else: 

120 more_pages = False 

121 

122 return result 

123 

124 

125@trace_function 

126def search_docs(term: str) -> dict: 

127 """ 

128 Fetches documentation from discourse from the doc category and 

129 a specific search term. 

130 

131 Parameters: 

132 search_term (str): The search term used to find relevant documentation. 

133 page (int): The page number of the search results to retrieve. 

134 see_all (bool, optional): If True, retrieves all available search 

135 results. If False (default), returns a limited number of results 

136 (5 posts and topics). 

137 

138 Returns: 

139 dict: A dictionary containing the retrieved dtopics. 

140 """ 

141 key = ("search-docs", {"q": term}) 

142 results = redis_cache.get(key, expected_type=list) 

143 if results: 

144 return results 

145 search_url = ( 

146 f"{DOCS_URL}/_/api/v3/search/?q=project%3Acanonical-juju+{term}" 

147 ) 

148 

149 resp = requests.get(search_url) 

150 data = resp.json() 

151 

152 results = data.get("results", []) 

153 redis_cache.set(key, results, ttl=3600) 

154 

155 return results 

156 

157 

158@trace_function 

159def search_charms(term: str): 

160 key = ("search-charms", {"q": term}) 

161 charms = redis_cache.get(key, expected_type=list) 

162 if charms: 

163 return charms 

164 charms = [ 

165 parse_package_for_card(package) 

166 for package in publisher_gateway.find( 

167 term, type="charm", fields=SEARCH_FIELDS 

168 )["results"] 

169 ] 

170 redis_cache.set(key, charms, ttl=3600) 

171 return charms 

172 

173 

174@trace_function 

175def search_bundles(term: str): 

176 key = ("search-bundles", {"q": term}) 

177 bundles = redis_cache.get(key, expected_type=list) 

178 if bundles: 

179 return bundles 

180 bundles = [ 

181 parse_package_for_card(package) 

182 for package in publisher_gateway.find( 

183 term, type="bundle", fields=SEARCH_FIELDS 

184 )["results"] 

185 ] 

186 redis_cache.set(key, bundles, ttl=3600) 

187 return bundles