review.fuel-infra Code Review - packages/trusty/python-django-compressor.git/blob - python-django-compressor/compressor/management/commands/compress.py

1 # flake8: noqa

2 import os

3 import sys

4

5 from collections import OrderedDict

6 from fnmatch import fnmatch

7 from optparse import make_option

8 from importlib import import_module

9

10 import django

11 from django.core.management.base import BaseCommand, CommandError

12 import django.template

13 from django.template import Context

14 from django.utils import six

15 from django.template.loader import get_template # noqa Leave this in to preload template locations

16 from django.template.utils import InvalidTemplateEngineError

17 from django.template import engines

18

19 from compressor.cache import get_offline_hexdigest, write_offline_manifest

20 from compressor.conf import settings

21 from compressor.exceptions import (OfflineGenerationError, TemplateSyntaxError,

22 TemplateDoesNotExist)

23 from compressor.templatetags.compress import CompressorNode

24 from compressor.utils import get_mod_func

25

26 if six.PY3:

27 # there is an 'io' module in python 2.6+, but io.StringIO does not

28 # accept regular strings, just unicode objects

29 from io import StringIO

30 else:

31 try:

32 from cStringIO import StringIO

33 except ImportError:

34 from StringIO import StringIO

35

36

37 class Command(BaseCommand):

38 help = "Compress content outside of the request/response cycle"

39 option_list = BaseCommand.option_list + (

40 make_option('--extension', '-e', action='append', dest='extensions',

41 help='The file extension(s) to examine (default: ".html", '

42 'separate multiple extensions with commas, or use -e '

43 'multiple times)'),

44 make_option('-f', '--force', default=False, action='store_true',

45 help="Force the generation of compressed content even if the "

46 "COMPRESS_ENABLED setting is not True.", dest='force'),

47 make_option('--follow-links', default=False, action='store_true',

48 help="Follow symlinks when traversing the COMPRESS_ROOT "

49 "(which defaults to STATIC_ROOT). Be aware that using this "

50 "can lead to infinite recursion if a link points to a parent "

51 "directory of itself.", dest='follow_links'),

52 make_option('--engine', default="django", action="store",

53 help="Specifies the templating engine. jinja2 or django",

54 dest="engine"),

55 )

56

57 def get_loaders(self):

58 template_source_loaders = []

59 for e in engines.all():

60 if hasattr(e, 'engine'):

61 template_source_loaders.extend(

62 e.engine.get_template_loaders(e.engine.loaders))

63 loaders = []

64 # If template loader is CachedTemplateLoader, return the loaders

65 # that it wraps around. So if we have

66 # TEMPLATE_LOADERS = (

67 # ('django.template.loaders.cached.Loader', (

68 # 'django.template.loaders.filesystem.Loader',

69 # 'django.template.loaders.app_directories.Loader',

70 # )),

71 # )

72 # The loaders will return django.template.loaders.filesystem.Loader

73 # and django.template.loaders.app_directories.Loader

74 # The cached Loader and similar ones include a 'loaders' attribute

75 # so we look for that.

76 for loader in template_source_loaders:

77 if hasattr(loader, 'loaders'):

78 loaders.extend(loader.loaders)

79 else:

80 loaders.append(loader)

81 return loaders

82

83 def __get_parser(self, engine):

84 if engine == "jinja2":

85 from compressor.offline.jinja2 import Jinja2Parser

86 env = settings.COMPRESS_JINJA2_GET_ENVIRONMENT()

87 parser = Jinja2Parser(charset=settings.FILE_CHARSET, env=env)

88 elif engine == "django":

89 from compressor.offline.django import DjangoParser

90 parser = DjangoParser(charset=settings.FILE_CHARSET)

91 else:

92 raise OfflineGenerationError("Invalid templating engine specified.")

93

94 return parser

95

96 def compress(self, log=None, **options):

97 """

98 Searches templates containing 'compress' nodes and compresses them

99 "offline" -- outside of the request/response cycle.

100

101 The result is cached with a cache-key derived from the content of the

102 compress nodes (not the content of the possibly linked files!).

103 """

104 engine = options.get("engine", "django")

105 extensions = options.get('extensions')

106 extensions = self.handle_extensions(extensions or ['html'])

107 verbosity = int(options.get("verbosity", 0))

108 if not log:

109 log = StringIO()

110 if not settings.TEMPLATE_LOADERS:

111 raise OfflineGenerationError("No template loaders defined. You "

112 "must set TEMPLATE_LOADERS in your "

113 "settings.")

114 templates = set()

115 if engine == 'django':

116 paths = set()

117 for loader in self.get_loaders():

118 try:

119 module = import_module(loader.__module__)

120 get_template_sources = getattr(module,

121 'get_template_sources', None)

122 if get_template_sources is None:

123 get_template_sources = loader.get_template_sources

124 paths.update(str(origin) for origin in get_template_sources(''))

125 except (ImportError, AttributeError, TypeError):

126 # Yeah, this didn't work out so well, let's move on

127 pass

128

129 if not paths:

130 raise OfflineGenerationError("No template paths found. None of "

131 "the configured template loaders "

132 "provided template paths. See "

133 "https://docs.djangoproject.com/en/1.8/topics/templates/ "

134 "for more information on template "

135 "loaders.")

136 if verbosity > 1:

137 log.write("Considering paths:\n\t" + "\n\t".join(paths) + "\n")

138

139 for path in paths:

140 for root, dirs, files in os.walk(path,

141 followlinks=options.get('followlinks', False)):

142 templates.update(os.path.join(root, name)

143 for name in files if not name.startswith('.') and

144 any(fnmatch(name, "*%s" % glob) for glob in extensions))

145 elif engine == 'jinja2' and django.VERSION >= (1, 8):

146 env = settings.COMPRESS_JINJA2_GET_ENVIRONMENT()

147 if env and hasattr(env, 'list_templates'):

148 templates |= set([env.loader.get_source(env, template)[1] for template in

149 env.list_templates(filter_func=lambda _path:

150 os.path.splitext(_path)[-1] in extensions)])

151

152 if not templates:

153 raise OfflineGenerationError("No templates found. Make sure your "

154 "TEMPLATE_LOADERS and TEMPLATE_DIRS "

155 "settings are correct.")

156 if verbosity > 1:

157 log.write("Found templates:\n\t" + "\n\t".join(templates) + "\n")

158

159 parser = self.__get_parser(engine)

160 compressor_nodes = OrderedDict()

161 for template_name in templates:

162 try:

163 template = parser.parse(template_name)

164 except IOError: # unreadable file -> ignore

165 if verbosity > 0:

166 log.write("Unreadable template at: %s\n" % template_name)

167 continue

168 except TemplateSyntaxError as e: # broken template -> ignore

169 if verbosity > 0:

170 log.write("Invalid template %s: %s\n" % (template_name, e))

171 continue

172 except TemplateDoesNotExist: # non existent template -> ignore

173 if verbosity > 0:

174 log.write("Non-existent template at: %s\n" % template_name)

175 continue

176 except UnicodeDecodeError:

177 if verbosity > 0:

178 log.write("UnicodeDecodeError while trying to read "

179 "template %s\n" % template_name)

180 try:

181 nodes = list(parser.walk_nodes(template))

182 except (TemplateDoesNotExist, TemplateSyntaxError) as e:

183 # Could be an error in some base template

184 if verbosity > 0:

185 log.write("Error parsing template %s: %s\n" % (template_name, e))

186 continue

187 if nodes:

188 template.template_name = template_name

189 compressor_nodes.setdefault(template, []).extend(nodes)

190

191 if not compressor_nodes:

192 raise OfflineGenerationError(

193 "No 'compress' template tags found in templates."

194 "Try running compress command with --follow-links and/or"

195 "--extension=EXTENSIONS")

196

197 if verbosity > 0:

198 log.write("Found 'compress' tags in:\n\t" +

199 "\n\t".join((t.template_name

200 for t in compressor_nodes.keys())) + "\n")

201

202 contexts = settings.COMPRESS_OFFLINE_CONTEXT

203 if isinstance(contexts, six.string_types):

204 try:

205 module, function = get_mod_func(contexts)

206 contexts = getattr(import_module(module), function)()

207 except (AttributeError, ImportError, TypeError) as e:

208 raise ImportError("Couldn't import offline context function %s: %s" %

209 (settings.COMPRESS_OFFLINE_CONTEXT, e))

210 elif not isinstance(contexts, (list, tuple)):

211 contexts = [contexts]

212

213 log.write("Compressing... ")

214 block_count = context_count = 0

215 results = []

216 offline_manifest = OrderedDict()

217

218 for context_dict in contexts:

219 context_count += 1

220 init_context = parser.get_init_context(context_dict)

221

222 for template, nodes in compressor_nodes.items():

223 context = Context(init_context)

224 template._log = log

225 template._log_verbosity = verbosity

226

227 if not parser.process_template(template, context):

228 continue

229

230 for node in nodes:

231 context.push()

232 parser.process_node(template, context, node)

233 rendered = parser.render_nodelist(template, context, node)

234 key = get_offline_hexdigest(rendered)

235

236 if key in offline_manifest:

237 continue

238

239 try:

240 result = parser.render_node(template, context, node)

241 except Exception as e:

242 raise CommandError("An error occurred during rendering %s: "

243 "%s" % (template.template_name, e))

244 offline_manifest[key] = result

245 context.pop()

246 results.append(result)

247 block_count += 1

248

249 write_offline_manifest(offline_manifest)

250

251 log.write("done\nCompressed %d block(s) from %d template(s) for %d context(s).\n" %

252 (block_count, len(compressor_nodes), context_count))

253 return block_count, results

254

255 def handle_extensions(self, extensions=('html',)):

256 """

257 organizes multiple extensions that are separated with commas or

258 passed by using --extension/-e multiple times.

259

260 for example: running 'django-admin compress -e js,txt -e xhtml -a'

261 would result in an extension list: ['.js', '.txt', '.xhtml']

262

263 >>> handle_extensions(['.html', 'html,js,py,py,py,.py', 'py,.py'])

264 ['.html', '.js']

265 >>> handle_extensions(['.html, txt,.tpl'])

266 ['.html', '.tpl', '.txt']

267 """

268 ext_list = []

269 for ext in extensions:

270 ext_list.extend(ext.replace(' ', '').split(','))

271 for i, ext in enumerate(ext_list):

272 if not ext.startswith('.'):

273 ext_list[i] = '.%s' % ext_list[i]

274 return set(ext_list)

275

276 def handle(self, **options):

277 if not settings.COMPRESS_ENABLED and not options.get("force"):

278 raise CommandError(

279 "Compressor is disabled. Set the COMPRESS_ENABLED "

280 "setting or use --force to override.")

281 if not settings.COMPRESS_OFFLINE:

282 if not options.get("force"):

283 raise CommandError(

284 "Offline compression is disabled. Set "

285 "COMPRESS_OFFLINE or use the --force to override.")

286 self.compress(sys.stdout, **options)

287

288

289

290 Command.requires_system_checks = False