From 0e7b2a6da5836a95fecebed5db0f009ed3eddf66 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Thu, 3 Sep 2009 13:58:39 -0300 Subject: [PATCH] write header line by default when using csv exporter --HG-- extra : rebase_source : 2d2d7153dde5e3f77e682e16d2e4408f732f234e --- docs/topics/exporters.rst | 8 +++---- scrapy/contrib/exporter/__init__.py | 20 +++++++++++------- scrapy/tests/test_contrib_exporter.py | 30 +++++++++++++++++++++------ 3 files changed, 40 insertions(+), 18 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index f0ccc4c4e..15458dddc 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -244,7 +244,7 @@ XmlItemExporter CsvItemExporter --------------- -.. class:: CsvItemExporter(file, include_headers_line=False, \**kwargs) +.. class:: CsvItemExporter(file, include_headers_line=True, \**kwargs) Exports Items in CSV format to the given file-like object. If the :attr:`fields_to_export` attribute is set, it will be used to define the @@ -254,9 +254,8 @@ CsvItemExporter :param file: the file-like object to use for exporting the data. :param include_headers_line: If enabled, makes the exporter output a header - line with the field names taken from - :attr:`BaseItemExporter.fields_to_export` so that attribute must also be - set in order to work (otherwise it raises a :exc:`RuntimeError`) + line with the field names taken from + :attr:`BaseItemExporter.fields_to_export` or the first exported item fields. :type include_headers_line: boolean The additional keyword arguments of this constructor are passed to the @@ -266,6 +265,7 @@ CsvItemExporter A typical output of this exporter would be:: + product,price Color TV,1200 DVD player,200 diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/contrib/exporter/__init__.py index 80dc704e4..3dbeb235f 100644 --- a/scrapy/contrib/exporter/__init__.py +++ b/scrapy/contrib/exporter/__init__.py @@ -101,25 +101,29 @@ class XmlItemExporter(BaseItemExporter): class CsvItemExporter(BaseItemExporter): - def __init__(self, file, include_headers_line=False, **kwargs): + def __init__(self, file, include_headers_line=True, **kwargs): self._configure(kwargs, dont_fail=True) self.include_headers_line = include_headers_line self.csv_writer = csv.writer(file, **kwargs) - - def start_exporting(self): - if self.include_headers_line: - if not self.fields_to_export: - raise RuntimeError("You must set fields_to_export in order" + \ - " to use include_headers_line") - self.csv_writer.writerow(self.fields_to_export) + self._header_written = False def export_item(self, item): + if not self._header_written: + self._write_header(item) + fields = self._get_serialized_fields(item, default_value='', \ include_empty=True) values = [x[1] for x in fields] self.csv_writer.writerow(values) + def _write_header(self, item): + self._header_written = True + if self.include_headers_line: + if not self.fields_to_export: + self.fields_to_export = item.fields.keys() + self.csv_writer.writerow(self.fields_to_export) + class PickleItemExporter(BaseItemExporter): diff --git a/scrapy/tests/test_contrib_exporter.py b/scrapy/tests/test_contrib_exporter.py index d5cd1765f..9b91759c9 100644 --- a/scrapy/tests/test_contrib_exporter.py +++ b/scrapy/tests/test_contrib_exporter.py @@ -95,20 +95,38 @@ class CsvItemExporterTest(BaseItemExporterTest): return CsvItemExporter(self.output, **kwargs) def _check_output(self): - self.assertEqual(self.output.getvalue(), '22,John\xc2\xa3\r\n') + self.assertEqual(self.output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n') def test_header(self): output = StringIO() - ie = CsvItemExporter(output, include_headers_line=True) - self.assertRaises(RuntimeError, ie.start_exporting) - - ie = CsvItemExporter(output, include_headers_line=True, \ - fields_to_export=self.i.fields.keys()) + ie = CsvItemExporter(output, fields_to_export=self.i.fields.keys()) ie.start_exporting() ie.export_item(self.i) ie.finish_exporting() self.assertEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n') + output = StringIO() + ie = CsvItemExporter(output, fields_to_export=['age']) + ie.start_exporting() + ie.export_item(self.i) + ie.finish_exporting() + self.assertEqual(output.getvalue(), 'age\r\n22\r\n') + + output = StringIO() + ie = CsvItemExporter(output) + ie.start_exporting() + ie.export_item(self.i) + ie.export_item(self.i) + ie.finish_exporting() + self.assertEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n') + + output = StringIO() + ie = CsvItemExporter(output, include_headers_line=False) + ie.start_exporting() + ie.export_item(self.i) + ie.finish_exporting() + self.assertEqual(output.getvalue(), '22,John\xc2\xa3\r\n') + class XmlItemExporterTest(BaseItemExporterTest): -- GitLab