<html xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
<meta name="Generator" content="Microsoft Word 15 (filtered medium)">
<style><!--
/* Font Definitions */
@font-face
{font-family:Wingdings;
panose-1:5 0 0 0 0 0 0 0 0 0;}
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:DengXian;
panose-1:2 1 6 0 3 1 1 1 1 1;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;}
@font-face
{font-family:"\@DengXian";
panose-1:2 1 6 0 3 1 1 1 1 1;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0cm;
margin-bottom:.0001pt;
font-size:10.0pt;
font-family:"Calibri",sans-serif;}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:#0563C1;
text-decoration:underline;}
a:visited, span.MsoHyperlinkFollowed
{mso-style-priority:99;
color:#954F72;
text-decoration:underline;}
p.MsoListParagraph, li.MsoListParagraph, div.MsoListParagraph
{mso-style-priority:34;
margin-top:0cm;
margin-right:0cm;
margin-bottom:0cm;
margin-left:36.0pt;
margin-bottom:.0001pt;
font-size:10.0pt;
font-family:"Calibri",sans-serif;}
p.msonormal0, li.msonormal0, div.msonormal0
{mso-style-name:msonormal;
mso-margin-top-alt:auto;
margin-right:0cm;
mso-margin-bottom-alt:auto;
margin-left:0cm;
font-size:12.0pt;
font-family:"Times New Roman",serif;}
span.E-MailFormatvorlage19
{mso-style-type:personal;
font-family:"Calibri",sans-serif;
color:windowtext;}
span.E-MailFormatvorlage20
{mso-style-type:personal-reply;
font-family:"Calibri",sans-serif;
color:#1F497D;}
.MsoChpDefault
{mso-style-type:export-only;
font-size:10.0pt;}
@page WordSection1
{size:612.0pt 792.0pt;
margin:70.85pt 70.85pt 2.0cm 70.85pt;}
div.WordSection1
{page:WordSection1;}
/* List Definitions */
@list l0
{mso-list-id:931938421;
mso-list-template-ids:131771608;}
@list l0:level1
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:36.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level2
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:72.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level3
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:108.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level4
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:144.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level5
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:180.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level6
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:216.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level7
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:252.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level8
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:288.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level9
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:324.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1
{mso-list-id:1427073455;
mso-list-type:hybrid;
mso-list-template-ids:-1805218188 -98254246 134807555 134807557 134807553 134807555 134807557 134807553 134807555 134807557;}
@list l1:level1
{mso-level-start-at:3;
mso-level-number-format:bullet;
mso-level-text:-;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:21.0pt;
text-indent:-18.0pt;
font-family:"Calibri",sans-serif;
mso-fareast-font-family:"Times New Roman";}
@list l1:level2
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:57.0pt;
text-indent:-18.0pt;
font-family:"Courier New";}
@list l1:level3
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:93.0pt;
text-indent:-18.0pt;
font-family:Wingdings;}
@list l1:level4
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:129.0pt;
text-indent:-18.0pt;
font-family:Symbol;}
@list l1:level5
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:165.0pt;
text-indent:-18.0pt;
font-family:"Courier New";}
@list l1:level6
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:201.0pt;
text-indent:-18.0pt;
font-family:Wingdings;}
@list l1:level7
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:237.0pt;
text-indent:-18.0pt;
font-family:Symbol;}
@list l1:level8
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:273.0pt;
text-indent:-18.0pt;
font-family:"Courier New";}
@list l1:level9
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:309.0pt;
text-indent:-18.0pt;
font-family:Wingdings;}
ol
{margin-bottom:0cm;}
ul
{margin-bottom:0cm;}
--></style><!--[if gte mso 9]><xml>
<o:shapedefaults v:ext="edit" spidmax="1026" />
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext="edit">
<o:idmap v:ext="edit" data="1" />
</o:shapelayout></xml><![endif]-->
</head>
<body lang="DE" link="#0563C1" vlink="#954F72">
<div class="WordSection1">
<p class="MsoNormal"><span style="font-size:11.0pt;color:#1F497D">Hi Patrick,<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;color:#1F497D">makes 100% sense. Thanks for the heads-up and the ways around the “feature” ;-)<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;color:#1F497D">Maybe a note in the “Description” that the first record serves as model for the fields exported would help the un-initiated like myself:
<a href="https://metacpan.org/pod/Catmandu::Exporter::TSV">https://metacpan.org/pod/Catmandu::Exporter::TSV</a><o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;color:#1F497D">Best,<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;color:#1F497D">Martina<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US"><o:p> </o:p></span></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal"><b><span style="font-size:11.0pt">Von:</span></b><span style="font-size:11.0pt"> Patrick Hochstenbach <Patrick.Hochstenbach@UGent.be>
<br>
<b>Gesendet:</b> Freitag, 3. November 2023 16:58<br>
<b>An:</b> Siebert, Dr. Martina <Martina.Siebert@sbb.spk-berlin.de>; librecat-dev@lists.uni-bielefeld.de<br>
<b>Betreff:</b> Re: [librecat-dev] Catmandu: field not converted to TSV/CSV output if not present in first (how many?) input records<o:p></o:p></span></p>
</div>
</div>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal"><span lang="NL" style="font-size:14.0pt;mso-fareast-language:EN-US">Hello Martina,<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="NL" style="font-size:14.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:14.0pt;mso-fareast-language:EN-US">This is a known feature. Catmandu was created to work with very large files in streaming mode. This means that Catmandu doesn’t read the complete input data first to
see what kind of fields are available. The software needs to work with the data at hand.<br>
<br>
The JSON format allows that every record have different fields, and doesn’t care if ‘late-comer’ records have a different field layout. Formats such as TSV and CSV don’t have this feature: from the first record you need to tell the software what fields need
to be made available. Catmandu use two procedures:<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:14.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal" style="margin-left:21.0pt;text-indent:-18.0pt;mso-list:l1 level1 lfo3">
<![if !supportLists]><span lang="EN-US" style="font-size:14.0pt;mso-fareast-language:EN-US"><span style="mso-list:Ignore">-<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span lang="EN-US" style="font-size:14.0pt;mso-fareast-language:EN-US">If no information is provided, the field layout is guessed from the first record it receives.<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:21.0pt;text-indent:-18.0pt;mso-list:l1 level1 lfo3">
<![if !supportLists]><span lang="EN-US" style="font-size:14.0pt;mso-fareast-language:EN-US"><span style="mso-list:Ignore">-<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span lang="EN-US" style="font-size:14.0pt;mso-fareast-language:EN-US">One can provide information to Catmandu which fields one wants to see in the output.<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:14.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:14.0pt;mso-fareast-language:EN-US">For the latter Catmandu has the `--fields` option:<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:14.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:14.0pt;font-family:"Courier New";mso-fareast-language:EN-US">$ catmandu convert JSON to CSV –-fix my.fix –-fields ‘id,name,title,author’ < data.json<br>
<br>
</span><span lang="EN-US" style="font-size:14.0pt;mso-fareast-language:EN-US">As I am writing this, I see also a `--collect_fields 1` option, that does what you want. But first load all the data into memory.<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:14.0pt;font-family:"Courier New";mso-fareast-language:EN-US"><br>
</span><span lang="EN-US" style="font-size:14.0pt;mso-fareast-language:EN-US">BR<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:14.0pt;mso-fareast-language:EN-US">Patrick<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:14.0pt;font-family:"Courier New";mso-fareast-language:EN-US"><br>
</span><span lang="EN-US" style="font-size:14.0pt;mso-fareast-language:EN-US">PS: Do to outlook issues some hyphens and quotes may be lost in the email</span><span lang="EN-US" style="font-size:14.0pt;font-family:"Courier New";mso-fareast-language:EN-US"> <o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:14.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<div id="mail-editor-reference-message-container">
<div>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal" style="margin-bottom:12.0pt"><b><span style="font-size:12.0pt;color:black">From:
</span></b><span style="font-size:12.0pt;color:black"><a href="mailto:librecat-dev-bounces@lists.uni-bielefeld.de">librecat-dev-bounces@lists.uni-bielefeld.de</a> <<a href="mailto:librecat-dev-bounces@lists.uni-bielefeld.de">librecat-dev-bounces@lists.uni-bielefeld.de</a>>
on behalf of Siebert, Dr. Martina <<a href="mailto:Martina.Siebert@sbb.spk-berlin.de">Martina.Siebert@sbb.spk-berlin.de</a>><br>
<b>Date: </b>Friday, 3 November 2023 at 15:29<br>
<b>To: </b><a href="mailto:librecat-dev@lists.uni-bielefeld.de">librecat-dev@lists.uni-bielefeld.de</a> <<a href="mailto:librecat-dev@lists.uni-bielefeld.de">librecat-dev@lists.uni-bielefeld.de</a>><br>
<b>Subject: </b>[librecat-dev] Catmandu: field not converted to TSV/CSV output if not present in first (how many?) input records<o:p></o:p></span></p>
</div>
<p class="MsoNormal"><span style="font-size:11.0pt">Hello,<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"> <o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt">It seems not possible to produce a TSV/CSV output that includes fields that appear for the first time only much later in an input file. In the JSON export all is fine, but when exporting to TSV/CSV
the “late-comer” fields are missing. When I fake-add the field to the first record the TSV/CSV export is correct.</span><span style="font-size:11.0pt"><o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt">Is this a known bug? Can it be fixed?</span><span style="font-size:11.0pt"><o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"> </span><span style="font-size:11.0pt"><o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Best,<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Martina<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:8.0pt;font-family:"Arial",sans-serif">_____________________________________________</span><span style="font-size:11.0pt"><o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:9.0pt">Dr. Martina Siebert</span><span style="font-size:11.0pt"><o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:9.0pt">Ostasienabteilung | CrossAsia</span><span style="font-size:11.0pt"><o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:9.0pt">Staatsbibliothek zu Berlin – Preußischer Kulturbesitz</span><span style="font-size:11.0pt"><o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:9.0pt"> </span><span style="font-size:11.0pt"><o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:9.0pt"><a href="mailto:martina.siebert@sbb.spk-berlin.de">martina.siebert@sbb.spk-berlin.de</a></span><span style="font-size:11.0pt"><o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:9.0pt"><a href="http://www.staatsbibliothek-berlin.de/">www.staatsbibliothek-berlin.de</a></span><span style="font-size:11.0pt"><o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:9.0pt"> </span><span style="font-size:11.0pt"><o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:9.0pt">Im Rahmen der E-Mail-Kommunikation werden gegebenenfalls personenbezogene Daten verarbeitet.
<br>
Unsere Hinweise zum Datenschutz finden Sie hier: <a href="http://sbb.berlin/datenschutz" target="_short" title="Kurz-URL">
http://sbb.berlin/datenschutz</a></span><span style="font-size:11.0pt"><o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"> <o:p></o:p></span></p>
</div>
</div>
</div>
</body>
</html>