Coverage for tests/test_robots.py: 95%

1import os 

2import re 

3import unittest 

4 

5from protego import Protego 

6 

7from webapp.app import create_app 

8 

9ROBOTS_PATH = os.path.join(os.path.dirname(__file__), "..", "robots.txt") 

10BASE_URL = "https://snapcraft.io" 

11 

12 

13def parse_groups(text): 

14 groups = {} 

15 pending = [] 

16 in_rules = False 

17 

18 for raw_line in text.splitlines(): 

19 line = raw_line.split("#", 1)[0].strip() 

20 

21 if not line or ":" not in line: 

22 continue 

23 

24 key, value = (part.strip() for part in line.split(":", 1)) 

25 key = key.lower() 

26 

27 if key == "user-agent": 

28 if in_rules: 

29 pending = [] 

30 in_rules = False 

31 pending.append(value.lower()) 

32 groups.setdefault(value.lower(), []) 

33 elif key in ("allow", "disallow"): 

34 in_rules = True 

35 for agent in pending: 

36 groups[agent].append((key, value)) 

37 

38 return groups 

39 

40 

41def is_login_gated(view): 

42 seen = set() 

43 

44 while view is not None and id(view) not in seen: 

45 seen.add(id(view)) 

46 code = getattr(view, "__code__", None) 

47 if code is not None and code.co_name == "is_user_logged_in": 

48 return True 

49 view = getattr(view, "__wrapped__", None) 

50 

51 return False 

52 

53 

54def sample_path(rule): 

55 path = re.sub(r"<any\([^)]*\):[^>]+>", "snaps", rule) 

56 path = re.sub(r"<regex\([^)]*\):[^>]+>", "firefox", path) 

57 path = re.sub(r"<path:[^>]+>", "x/y", path) 

58 path = re.sub(r"<[^>]*snap_name>", "firefox", path) 

59 

60 return re.sub(r"<[^>]+>", "x", path) 

61 

62 

63class TestRobots(unittest.TestCase): 

64 def setUp(self): 

65 with open(ROBOTS_PATH) as robots_file: 

66 robots = robots_file.read() 

67 

68 self.robots = Protego.parse(robots) 

69 self.groups = parse_groups(robots) 

70 self.app = create_app(testing=True) 

71 

72 def can_fetch(self, agent, path): 

73 return self.robots.can_fetch(BASE_URL + path, agent) 

74 

75 def find_crawlable(self, paths): 

76 return [ 

77 (agent, path) 

78 for agent in self.groups 

79 for path in paths 

80 if self.can_fetch(agent, path) 

81 ] 

82 

83 def find_blocked(self, paths): 

84 return [ 

85 (agent, path) 

86 for agent in self.groups 

87 for path in paths 

88 if not self.can_fetch(agent, path) 

89 ] 

90 

91 def test_robots_txt_is_served_and_not_empty(self): 

92 response = self.app.test_client().get("/robots.txt") 

93 

94 self.assertEqual(response.status_code, 200) 

95 self.assertIn(b"User-Agent", response.data) 

96 

97 def test_login_gated_routes_are_disallowed_for_every_agent(self): 

98 gated = sorted( 

99 { 

100 sample_path(rule.rule) 

101 for rule in self.app.url_map.iter_rules() 

102 if is_login_gated(self.app.view_functions.get(rule.endpoint)) 

103 } 

104 ) 

105 

106 self.assertTrue(gated, "no login-gated routes found") 

107 

108 leaks = self.find_crawlable(gated) 

109 

110 self.assertEqual(leaks, [], f"login-gated routes crawlable: {leaks}") 

111 

112 def test_public_content_is_crawlable_for_every_agent(self): 

113 public_paths = [ 

114 "/", 

115 "/about", 

116 "/about/publish", 

117 "/about/listing", 

118 "/about/release", 

119 "/about/publicise", 

120 "/store", 

121 "/store/categories/games", 

122 "/blog/", 

123 "/docs/", 

124 "/build", 

125 "/iot", 

126 "/tutorials", 

127 "/firefox", 

128 "/publisher/canonical", 

129 ] 

130 

131 blocked = self.find_blocked(public_paths) 

132 

133 self.assertEqual( 

134 blocked, [], f"public content is not crawlable: {blocked}" 

135 ) 

136 

137 def test_snaps_named_after_reserved_paths_are_crawlable(self): 

138 snap_paths = [ 

139 "/accountable2you", 

140 "/administrative-assistant", 

141 "/searchsploit", 

142 "/search-helper-tool", 

143 "/build-and-measure", 

144 "/iot-manager", 

145 "/iotconnect", 

146 "/publisher-subscriber", 

147 "/store-admin", 

148 "/api-mocker-gateway", 

149 "/apilume", 

150 ] 

151 

152 blocked = self.find_blocked(snap_paths) 

153 

154 self.assertEqual( 

155 blocked, [], f"snap pages blocked by an unanchored rule: {blocked}" 

156 ) 

157 

158 def test_wildcard_rules_apply_to_every_named_agent(self): 

159 wildcard = set(self.groups["*"]) 

160 

161 for agent, rules in self.groups.items(): 

162 if agent == "*": 

163 continue 

164 

165 missing = sorted(wildcard - set(rules)) 

166 

167 self.assertEqual( 

168 missing, [], f"{agent} is missing rules from '*': {missing}" 

169 )