1 | # -*- coding: utf-8 -*- |
---|
2 | |
---|
3 | import re |
---|
4 | import types |
---|
5 | from datetime import datetime |
---|
6 | from itertools import chain, groupby |
---|
7 | from xlwt import Workbook, Formula |
---|
8 | |
---|
9 | from trac.core import Component, implements |
---|
10 | from trac.env import Environment |
---|
11 | from trac.mimeview.api import Context, IContentConverter |
---|
12 | from trac.resource import Resource, get_resource_url |
---|
13 | from trac.ticket.api import TicketSystem |
---|
14 | from trac.ticket.model import Ticket |
---|
15 | from trac.ticket.query import Query |
---|
16 | from trac.ticket.web_ui import TicketModule |
---|
17 | from trac.util import Ranges |
---|
18 | from trac.util.text import empty, unicode_urlencode |
---|
19 | from trac.web.api import IRequestFilter, RequestDone |
---|
20 | from trac.web.chrome import Chrome, add_link |
---|
21 | try: |
---|
22 | from trac.util.datefmt import from_utimestamp |
---|
23 | except ImportError: |
---|
24 | from datetime import timedelta |
---|
25 | from trac.util.datefmt import utc |
---|
26 | _epoc = datetime(1970, 1, 1, tzinfo=utc) |
---|
27 | from_utimestamp = lambda ts: _epoc + timedelta(seconds=ts or 0) |
---|
28 | |
---|
29 | from tracexceldownload.api import WorksheetWriter, get_workbook_content, \ |
---|
30 | get_literal |
---|
31 | from tracexceldownload.translation import _, dgettext, dngettext |
---|
32 | |
---|
33 | |
---|
34 | if hasattr(Environment, 'get_read_db'): |
---|
35 | _get_db = lambda env: env.get_read_db() |
---|
36 | else: |
---|
37 | _get_db = lambda env: env.get_db_cnx() |
---|
38 | |
---|
39 | |
---|
40 | def _tkt_id_conditions(column, tkt_ids): |
---|
41 | ranges = Ranges() |
---|
42 | ranges.appendrange(','.join(map(str, sorted(tkt_ids)))) |
---|
43 | condition = [] |
---|
44 | tkt_ids = [] |
---|
45 | for a, b in ranges.pairs: |
---|
46 | if a == b: |
---|
47 | tkt_ids.append(a) |
---|
48 | elif a + 1 == b: |
---|
49 | tkt_ids.extend((a, b)) |
---|
50 | else: |
---|
51 | condition.append('%s BETWEEN %d AND %d' % (column, a, b)) |
---|
52 | if tkt_ids: |
---|
53 | condition.append('%s IN (%s)' % (column, ','.join(map(str, tkt_ids)))) |
---|
54 | return ' OR '.join(condition) |
---|
55 | |
---|
56 | |
---|
57 | class BulkFetchTicket(Ticket): |
---|
58 | |
---|
59 | @classmethod |
---|
60 | def select(cls, env, tkt_ids): |
---|
61 | if not tkt_ids: |
---|
62 | return {} |
---|
63 | |
---|
64 | db = _get_db(env) |
---|
65 | fields = TicketSystem(env).get_ticket_fields() |
---|
66 | std_fields = [f['name'] for f in fields if not f.get('custom')] |
---|
67 | time_fields = [f['name'] for f in fields if f['type'] == 'time'] |
---|
68 | custom_fields = set(f['name'] for f in fields if f.get('custom')) |
---|
69 | cursor = db.cursor() |
---|
70 | tickets = {} |
---|
71 | |
---|
72 | cursor.execute('SELECT %s,id FROM ticket WHERE %s' % |
---|
73 | (','.join(std_fields), |
---|
74 | _tkt_id_conditions('id', tkt_ids))) |
---|
75 | for row in cursor: |
---|
76 | id = row[-1] |
---|
77 | values = {} |
---|
78 | for idx, field in enumerate(std_fields): |
---|
79 | value = row[idx] |
---|
80 | if field in time_fields: |
---|
81 | value = from_utimestamp(value) |
---|
82 | elif value is None: |
---|
83 | value = empty |
---|
84 | values[field] = value |
---|
85 | tickets[id] = (values, []) # values, changelog |
---|
86 | |
---|
87 | cursor.execute('SELECT ticket,name,value FROM ticket_custom ' |
---|
88 | 'WHERE %s ORDER BY ticket' % |
---|
89 | _tkt_id_conditions('ticket', tkt_ids)) |
---|
90 | for id, rows in groupby(cursor, lambda row: row[0]): |
---|
91 | if id not in tickets: |
---|
92 | continue |
---|
93 | values = {} |
---|
94 | for id, name, value in rows: |
---|
95 | if name in custom_fields: |
---|
96 | if value is None: |
---|
97 | value = empty |
---|
98 | values[name] = value |
---|
99 | tickets[id][0].update(values) |
---|
100 | |
---|
101 | cursor.execute('SELECT ticket,time,author,field,oldvalue,newvalue ' |
---|
102 | 'FROM ticket_change WHERE %s ORDER BY ticket,time' % |
---|
103 | _tkt_id_conditions('ticket', tkt_ids)) |
---|
104 | for id, rows in groupby(cursor, lambda row: row[0]): |
---|
105 | if id not in tickets: |
---|
106 | continue |
---|
107 | tickets[id][1].extend( |
---|
108 | (from_utimestamp(t), author, field, oldvalue or '', |
---|
109 | newvalue or '', 1) |
---|
110 | for id, t, author, field, oldvalue, newvalue in rows) |
---|
111 | |
---|
112 | return dict((id, cls(env, id, values=values, changelog=changelog, |
---|
113 | fields=fields, time_fields=time_fields)) |
---|
114 | for id, (values, changelog) in tickets.iteritems()) |
---|
115 | |
---|
116 | def __init__(self, env, tkt_id=None, db=None, version=None, values=None, |
---|
117 | changelog=None, fields=None, time_fields=None): |
---|
118 | self.env = env |
---|
119 | if tkt_id is not None: |
---|
120 | tkt_id = int(tkt_id) |
---|
121 | self.resource = Resource('ticket', tkt_id, version) |
---|
122 | self.fields = fields |
---|
123 | self.time_fields = time_fields |
---|
124 | self.id = tkt_id |
---|
125 | self._values = values |
---|
126 | self.values = values.copy() |
---|
127 | self._changelog = changelog |
---|
128 | self._old = {} |
---|
129 | |
---|
130 | def _fetch_ticket(self, tkt_id, db=None): |
---|
131 | self.values = self._values.copy() |
---|
132 | |
---|
133 | def get_changelog(self, when=None, db=None): |
---|
134 | return self._changelog[:] |
---|
135 | |
---|
136 | |
---|
137 | class ExcelTicketModule(Component): |
---|
138 | |
---|
139 | implements(IContentConverter) |
---|
140 | |
---|
141 | def get_supported_conversions(self): |
---|
142 | yield ('excel', _("Excel"), 'xls', |
---|
143 | 'trac.ticket.Query', 'application/vnd.ms-excel', 8) |
---|
144 | yield ('excel-history', _("Excel including history"), 'xls', |
---|
145 | 'trac.ticket.Query', 'application/vnd.ms-excel', 8) |
---|
146 | yield ('excel-history', _("Excel including history"), 'xls', |
---|
147 | 'trac.ticket.Ticket', 'application/vnd.ms-excel', 8) |
---|
148 | |
---|
149 | def convert_content(self, req, mimetype, content, key): |
---|
150 | if key == 'excel': |
---|
151 | return self._convert_query(req, content) |
---|
152 | if key == 'excel-history': |
---|
153 | kwargs = {} |
---|
154 | if isinstance(content, Ticket): |
---|
155 | content = Query.from_string(self.env, 'id=%d' % content.id) |
---|
156 | kwargs['sheet_query'] = False |
---|
157 | kwargs['sheet_history'] = True |
---|
158 | else: |
---|
159 | kwargs['sheet_query'] = True |
---|
160 | kwargs['sheet_history'] = True |
---|
161 | return self._convert_query(req, content, **kwargs) |
---|
162 | |
---|
163 | def _convert_query(self, req, query, sheet_query=True, |
---|
164 | sheet_history=False): |
---|
165 | # no paginator |
---|
166 | query.max = 0 |
---|
167 | query.has_more_pages = False |
---|
168 | query.offset = 0 |
---|
169 | db = _get_db(self.env) |
---|
170 | |
---|
171 | # extract all fields except custom fields |
---|
172 | custom_fields = [f['name'] for f in query.fields if f.get('custom')] |
---|
173 | cols = ['id'] |
---|
174 | cols.extend(f['name'] for f in query.fields |
---|
175 | if f['name'] not in custom_fields) |
---|
176 | cols.extend(name for name in ('time', 'changetime') |
---|
177 | if name not in cols) |
---|
178 | query.cols = cols |
---|
179 | |
---|
180 | # prevent "SELECT COUNT(*)" query |
---|
181 | saved_count_prop = query._count |
---|
182 | try: |
---|
183 | query._count = types.MethodType(lambda self, sql, args, db=None: 0, |
---|
184 | query, query.__class__) |
---|
185 | tickets = query.execute(req, db) |
---|
186 | query.num_items = len(tickets) |
---|
187 | finally: |
---|
188 | query._count = saved_count_prop |
---|
189 | |
---|
190 | # add custom fields to avoid error to join many tables |
---|
191 | self._fill_custom_fields(tickets, query.fields, custom_fields, db) |
---|
192 | |
---|
193 | context = Context.from_request(req, 'query', absurls=True) |
---|
194 | cols.extend([name for name in custom_fields if name not in cols]) |
---|
195 | data = query.template_data(context, tickets) |
---|
196 | |
---|
197 | book = Workbook(encoding='utf-8') |
---|
198 | if sheet_query: |
---|
199 | self._create_sheet_query(req, context, data, book) |
---|
200 | if sheet_history: |
---|
201 | self._create_sheet_history(req, context, data, book) |
---|
202 | return get_workbook_content(book), 'application/vnd.ms-excel' |
---|
203 | |
---|
204 | def _fill_custom_fields(self, tickets, fields, custom_fields, db): |
---|
205 | if not tickets or not custom_fields: |
---|
206 | return |
---|
207 | fields = dict((f['name'], f) for f in fields) |
---|
208 | tickets = dict((int(ticket['id']), ticket) for ticket in tickets) |
---|
209 | query = "SELECT ticket,name,value " \ |
---|
210 | "FROM ticket_custom WHERE %s ORDER BY ticket" % \ |
---|
211 | _tkt_id_conditions('ticket', tickets) |
---|
212 | |
---|
213 | cursor = db.cursor() |
---|
214 | cursor.execute(query) |
---|
215 | for id, name, value in cursor: |
---|
216 | if id not in tickets: |
---|
217 | continue |
---|
218 | f = fields.get(name) |
---|
219 | if f and f['type'] == 'checkbox': |
---|
220 | try: |
---|
221 | value = bool(int(value)) |
---|
222 | except (TypeError, ValueError): |
---|
223 | value = False |
---|
224 | tickets[id][name] = value |
---|
225 | |
---|
226 | def _create_sheet_query(self, req, context, data, book): |
---|
227 | sheet = book.add_sheet(dgettext('messages', 'Custom Query')) |
---|
228 | writer = WorksheetWriter(sheet, req) |
---|
229 | query = data['query'] |
---|
230 | groups = data['groups'] |
---|
231 | fields = data['fields'] |
---|
232 | headers = data['headers'] |
---|
233 | |
---|
234 | writer.write_row([( |
---|
235 | u'%s (%s)' % (dgettext('messages', 'Custom Query'), |
---|
236 | dngettext('messages', '%(num)s match', |
---|
237 | '%(num)s matches', query.num_items)), |
---|
238 | 'header', -1, -1)]) |
---|
239 | for groupname, results in groups: |
---|
240 | if groupname: |
---|
241 | cell = fields[query.group]['label'] + ' ' |
---|
242 | if query.group in ('owner', 'reporter'): |
---|
243 | cell += Chrome(self.env).format_author(req, groupname) |
---|
244 | else: |
---|
245 | cell += groupname |
---|
246 | cell += ' (%s)' % dngettext('messages', '%(num)s match', |
---|
247 | '%(num)s matches', len(results)) |
---|
248 | writer.write_row([(cell, 'header2', -1, -1)]) |
---|
249 | |
---|
250 | writer.write_row( |
---|
251 | (header['label'], 'thead', None, None) |
---|
252 | for idx, header in enumerate(headers)) |
---|
253 | |
---|
254 | for result in results: |
---|
255 | ticket_context = context('ticket', result['id']) |
---|
256 | if 'TICKET_VIEW' not in req.perm(ticket_context.resource): |
---|
257 | continue |
---|
258 | cells = [] |
---|
259 | for idx, header in enumerate(headers): |
---|
260 | name = header['name'] |
---|
261 | value, style, width, line = self._get_cell_data( |
---|
262 | name, result.get(name), req, ticket_context, writer) |
---|
263 | cells.append((value, style, width, line)) |
---|
264 | writer.write_row(cells) |
---|
265 | writer.move_row() |
---|
266 | |
---|
267 | writer.set_col_widths() |
---|
268 | |
---|
269 | def _create_sheet_history(self, req, context, data, book): |
---|
270 | sheet = book.add_sheet(dgettext("messages", "Change History")) |
---|
271 | writer = WorksheetWriter(sheet, req) |
---|
272 | |
---|
273 | groups = data['groups'] |
---|
274 | headers = [header for header in data['headers'] |
---|
275 | if header['name'] not in ('id', 'time', 'changetime')] |
---|
276 | headers[0:0] = [ |
---|
277 | {'name': 'id', 'label': dgettext("messages", "Ticket")}, |
---|
278 | {'name': 'time', 'label': dgettext("messages", "Time")}, |
---|
279 | {'name': 'author', 'label': dgettext("messages", "Author")}, |
---|
280 | {'name': 'comment', 'label': dgettext("messages", "Comment")}, |
---|
281 | ] |
---|
282 | |
---|
283 | writer.write_row( |
---|
284 | (header['label'], 'thead', None, None) |
---|
285 | for idx, header in enumerate(headers)) |
---|
286 | |
---|
287 | tkt_ids = [result['id'] |
---|
288 | for result in chain(*[results for groupname, results |
---|
289 | in groups])] |
---|
290 | tickets = BulkFetchTicket.select(self.env, tkt_ids) |
---|
291 | |
---|
292 | mod = TicketModule(self.env) |
---|
293 | for result in chain(*[results for groupname, results in groups]): |
---|
294 | id = result['id'] |
---|
295 | ticket = tickets[id] |
---|
296 | ticket_context = context('ticket', id) |
---|
297 | if 'TICKET_VIEW' not in req.perm(ticket_context.resource): |
---|
298 | continue |
---|
299 | values = ticket.values.copy() |
---|
300 | changes = [] |
---|
301 | |
---|
302 | for change in mod.rendered_changelog_entries(req, ticket): |
---|
303 | if change['permanent']: |
---|
304 | changes.append(change) |
---|
305 | for change in reversed(changes): |
---|
306 | change['values'] = values |
---|
307 | values = values.copy() |
---|
308 | for name, field in change['fields'].iteritems(): |
---|
309 | if name in values: |
---|
310 | values[name] = field['old'] |
---|
311 | changes[0:0] = [{'date': ticket.time_created, 'fields': {}, |
---|
312 | 'values': values, 'cnum': None, |
---|
313 | 'comment': '', 'author': ticket['reporter']}] |
---|
314 | |
---|
315 | for change in changes: |
---|
316 | cells = [] |
---|
317 | for idx, header in enumerate(headers): |
---|
318 | name = header['name'] |
---|
319 | if name == 'id': |
---|
320 | value = id |
---|
321 | elif name == 'time': |
---|
322 | value = change.get('date', '') |
---|
323 | elif name == 'comment': |
---|
324 | value = change.get('comment', '') |
---|
325 | elif name == 'author': |
---|
326 | value = change.get('author', '') |
---|
327 | else: |
---|
328 | value = change['values'].get(name, '') |
---|
329 | value, style, width, line = \ |
---|
330 | self._get_cell_data(name, value, req, |
---|
331 | ticket_context, writer) |
---|
332 | if name in change['fields']: |
---|
333 | style = '%s:change' % style |
---|
334 | cells.append((value, style, width, line)) |
---|
335 | writer.write_row(cells) |
---|
336 | |
---|
337 | writer.set_col_widths() |
---|
338 | |
---|
339 | def _get_cell_data(self, name, value, req, context, writer): |
---|
340 | if name == 'id': |
---|
341 | url = self.env.abs_href.ticket(value) |
---|
342 | value = '#%d' % value |
---|
343 | width = len(value) |
---|
344 | value = Formula('HYPERLINK("%s",%s)' % (url, get_literal(value))) |
---|
345 | return value, 'id', width, 1 |
---|
346 | |
---|
347 | if isinstance(value, datetime): |
---|
348 | return value, '[datetime]', None, None |
---|
349 | |
---|
350 | if value and name in ('reporter', 'owner'): |
---|
351 | value = Chrome(self.env).format_author(req, value) |
---|
352 | return value, name, None, None |
---|
353 | |
---|
354 | if name == 'cc': |
---|
355 | value = Chrome(self.env).format_emails(context, value) |
---|
356 | return value, name, None, None |
---|
357 | |
---|
358 | if name == 'milestone': |
---|
359 | url = self.env.abs_href.milestone(value) |
---|
360 | width, line = writer.get_metrics(value) |
---|
361 | value = Formula('HYPERLINK("%s",%s)' % (url, get_literal(value))) |
---|
362 | return value, name, width, line |
---|
363 | |
---|
364 | return value, name, None, None |
---|
365 | |
---|
366 | |
---|
367 | class ExcelReportModule(Component): |
---|
368 | |
---|
369 | implements(IRequestFilter) |
---|
370 | |
---|
371 | _PATH_INFO_MATCH = re.compile(r'/report/[0-9]+').match |
---|
372 | |
---|
373 | def pre_process_request(self, req, handler): |
---|
374 | if self._PATH_INFO_MATCH(req.path_info) \ |
---|
375 | and req.args.get('format') == 'xls' \ |
---|
376 | and handler.__class__.__name__ == 'ReportModule': |
---|
377 | req.args['max'] = 0 |
---|
378 | return handler |
---|
379 | |
---|
380 | def post_process_request(self, req, template, data, content_type): |
---|
381 | if template == 'report_view.html' and req.args.get('id'): |
---|
382 | format = req.args.get('format') |
---|
383 | if format == 'xls': |
---|
384 | resource = Resource('report', req.args['id']) |
---|
385 | data['context'] = Context.from_request(req, resource, |
---|
386 | absurls=True) |
---|
387 | self._convert_report(req, data) |
---|
388 | elif not format: |
---|
389 | self._add_alternate_links(req) |
---|
390 | return template, data, content_type |
---|
391 | |
---|
392 | def _convert_report(self, req, data): |
---|
393 | book = Workbook(encoding='utf-8') |
---|
394 | sheet = book.add_sheet(dgettext('messages', 'Report')) |
---|
395 | writer = WorksheetWriter(sheet, req) |
---|
396 | |
---|
397 | writer.write_row([( |
---|
398 | '%s (%s)' % (data['title'], |
---|
399 | dngettext('messages', '%(num)s match', |
---|
400 | '%(num)s matches', data['numrows'])), |
---|
401 | 'header', -1, -1)]) |
---|
402 | |
---|
403 | for value_for_group, row_group in data['row_groups']: |
---|
404 | writer.move_row() |
---|
405 | |
---|
406 | if value_for_group and len(row_group): |
---|
407 | writer.write_row([( |
---|
408 | '%s (%s)' % (value_for_group, |
---|
409 | dngettext('messages', '%(num)s match', |
---|
410 | '%(num)s matches', len(row_group))), |
---|
411 | 'header2', -1, -1)]) |
---|
412 | for header_group in data['header_groups']: |
---|
413 | writer.write_row([ |
---|
414 | (header['title'], 'thead', None, None) |
---|
415 | for header in header_group |
---|
416 | if not header['hidden']]) |
---|
417 | |
---|
418 | for row in row_group: |
---|
419 | for cell_group in row['cell_groups']: |
---|
420 | cells = [] |
---|
421 | for cell in cell_group: |
---|
422 | cell_header = cell['header'] |
---|
423 | if cell_header['hidden']: |
---|
424 | continue |
---|
425 | col = cell_header['col'].strip('_').lower() |
---|
426 | value, style, width, line = \ |
---|
427 | self._get_cell_data(req, col, cell, row, writer) |
---|
428 | cells.append((value, style, width, line)) |
---|
429 | writer.write_row(cells) |
---|
430 | |
---|
431 | writer.set_col_widths() |
---|
432 | |
---|
433 | content = get_workbook_content(book) |
---|
434 | req.send_response(200) |
---|
435 | req.send_header('Content-Type', 'application/vnd.ms-excel') |
---|
436 | req.send_header('Content-Length', len(content)) |
---|
437 | req.send_header('Content-Disposition', |
---|
438 | 'filename=report_%s.xls' % req.args['id']) |
---|
439 | req.end_headers() |
---|
440 | req.write(content) |
---|
441 | raise RequestDone |
---|
442 | |
---|
443 | def _get_cell_data(self, req, col, cell, row, writer): |
---|
444 | value = cell['value'] |
---|
445 | |
---|
446 | if col == 'report': |
---|
447 | url = self.env.abs_href.report(value) |
---|
448 | width, line = writer.get_metrics(value) |
---|
449 | value = Formula('HYPERLINK("%s",%s)' % (url, get_literal(value))) |
---|
450 | return value, col, width, line |
---|
451 | |
---|
452 | if col in ('ticket', 'id'): |
---|
453 | value = '#%s' % cell['value'] |
---|
454 | url = get_resource_url(self.env, row['resource'], self.env.abs_href) |
---|
455 | width = len(value) |
---|
456 | value = Formula('HYPERLINK("%s",%s)' % (url, get_literal(value))) |
---|
457 | return value, 'id', width, 1 |
---|
458 | |
---|
459 | if col == 'milestone': |
---|
460 | url = self.env.abs_href.milestone(value) |
---|
461 | width, line = writer.get_metrics(value) |
---|
462 | value = Formula('HYPERLINK("%s",%s)' % (url, get_literal(value))) |
---|
463 | return value, col, width, line |
---|
464 | |
---|
465 | if col == 'time': |
---|
466 | if isinstance(value, basestring) and value.isdigit(): |
---|
467 | value = from_utimestamp(long(value)) |
---|
468 | return value, '[time]', None, None |
---|
469 | elif col in ('date', 'created', 'modified'): |
---|
470 | if isinstance(value, basestring) and value.isdigit(): |
---|
471 | value = from_utimestamp(long(value)) |
---|
472 | return value, '[date]', None, None |
---|
473 | elif col == 'datetime': |
---|
474 | if isinstance(value, basestring) and value.isdigit(): |
---|
475 | value = from_utimestamp(long(value)) |
---|
476 | return value, '[datetime]', None, None |
---|
477 | |
---|
478 | width, line = writer.get_metrics(value) |
---|
479 | return value, col, width, line |
---|
480 | |
---|
481 | def _add_alternate_links(self, req): |
---|
482 | params = {} |
---|
483 | for arg in req.args.keys(): |
---|
484 | if not arg.isupper(): |
---|
485 | continue |
---|
486 | params[arg] = req.args.get(arg) |
---|
487 | if 'USER' not in params: |
---|
488 | params['USER'] = req.authname |
---|
489 | if 'sort' in req.args: |
---|
490 | params['sort'] = req.args['sort'] |
---|
491 | if 'asc' in req.args: |
---|
492 | params['asc'] = req.args['asc'] |
---|
493 | href = '' |
---|
494 | if params: |
---|
495 | href = '&' + unicode_urlencode(params) |
---|
496 | add_link(req, 'alternate', '?format=xls' + href, _("Excel"), |
---|
497 | 'application/vnd.ms-excel') |
---|