orchard

  1// docx, xlsx and pptx.
  2//
  3// All three are a zip of xml, so they need no dependency and no subprocess. The
  4// parsing is deliberately shallow: what a model wants out of a spreadsheet is
  5// the values in row and column order, not the formatting, and everything these
  6// formats carry beyond that is noise it would have to read past.
  7package main
  8
  9import (
 10	"archive/zip"
 11	"bytes"
 12	"encoding/xml"
 13	"fmt"
 14	"io"
 15	"sort"
 16	"strconv"
 17	"strings"
 18)
 19
 20// officeKind names the format from the parts inside the zip rather than the
 21// extension, since all three look identical from the outside and a renamed file
 22// is common.
 23func officeKind(zr *zip.Reader) string {
 24	for _, f := range zr.File {
 25		switch {
 26		case f.Name == "word/document.xml":
 27			return "docx"
 28		case f.Name == "xl/workbook.xml":
 29			return "xlsx"
 30		case strings.HasPrefix(f.Name, "ppt/slides/slide"):
 31			return "pptx"
 32		}
 33	}
 34	return ""
 35}
 36
 37func officeText(raw []byte) (string, string, error) {
 38	zr, err := zip.NewReader(bytes.NewReader(raw), int64(len(raw)))
 39	if err != nil {
 40		return "", "", fmt.Errorf("this file is not readable as an office document")
 41	}
 42	kind := officeKind(zr)
 43	switch kind {
 44	case "docx":
 45		text, err := docxText(zr)
 46		return text, kind, err
 47	case "xlsx":
 48		text, err := xlsxText(zr)
 49		return text, kind, err
 50	case "pptx":
 51		text, err := pptxText(zr)
 52		return text, kind, err
 53	}
 54	return "", "", fmt.Errorf("this file is a zip but not a document")
 55}
 56
 57func openPart(zr *zip.Reader, name string) ([]byte, error) {
 58	for _, f := range zr.File {
 59		if f.Name == name {
 60			rc, err := f.Open()
 61			if err != nil {
 62				return nil, err
 63			}
 64			defer rc.Close()
 65			return io.ReadAll(io.LimitReader(rc, 64<<20))
 66		}
 67	}
 68	return nil, fmt.Errorf("%s is missing", name)
 69}
 70
 71// runsToText walks the xml pulling out the text runs and turning the elements
 72// that mean a line break into one. Decoding by token rather than into a struct
 73// keeps this indifferent to the parts of the schema it does not care about.
 74func runsToText(data []byte, breakOn map[string]bool) string {
 75	dec := xml.NewDecoder(bytes.NewReader(data))
 76	var sb strings.Builder
 77	inText := false
 78	for {
 79		tok, err := dec.Token()
 80		if err != nil {
 81			break
 82		}
 83		switch t := tok.(type) {
 84		case xml.StartElement:
 85			if t.Name.Local == "t" {
 86				inText = true
 87			}
 88		case xml.EndElement:
 89			if t.Name.Local == "t" {
 90				inText = false
 91			}
 92			if breakOn[t.Name.Local] {
 93				sb.WriteString("\n")
 94			}
 95		case xml.CharData:
 96			if inText {
 97				sb.Write(t)
 98			}
 99		}
100	}
101	return collapseBlankLines(sb.String())
102}
103
104func docxText(zr *zip.Reader) (string, error) {
105	data, err := openPart(zr, "word/document.xml")
106	if err != nil {
107		return "", fmt.Errorf("this docx has no document part")
108	}
109	// A paragraph and a table row both end a line, and a cell does not, so a
110	// table comes out one row per line rather than one word per line.
111	text := runsToText(data, map[string]bool{"p": true, "tr": true, "br": true})
112	if strings.TrimSpace(text) == "" {
113		return "", fmt.Errorf("this docx has no text in it")
114	}
115	return text, nil
116}
117
118func pptxText(zr *zip.Reader) (string, error) {
119	names := []string{}
120	for _, f := range zr.File {
121		if strings.HasPrefix(f.Name, "ppt/slides/slide") && strings.HasSuffix(f.Name, ".xml") {
122			names = append(names, f.Name)
123		}
124	}
125	// Zip order is not slide order, and slide10 sorts before slide2 as a
126	// string, so they are ordered by the number in the name.
127	sort.Slice(names, func(i, j int) bool { return slideNum(names[i]) < slideNum(names[j]) })
128
129	var sb strings.Builder
130	for i, name := range names {
131		data, err := openPart(zr, name)
132		if err != nil {
133			continue
134		}
135		fmt.Fprintf(&sb, "--- slide %d ---\n", i+1)
136		sb.WriteString(runsToText(data, map[string]bool{"p": true, "br": true}))
137		sb.WriteString("\n")
138	}
139	out := collapseBlankLines(sb.String())
140	if strings.TrimSpace(out) == "" {
141		return "", fmt.Errorf("this pptx has no text in it")
142	}
143	return out, nil
144}
145
146func slideNum(name string) int {
147	base := strings.TrimSuffix(strings.TrimPrefix(name, "ppt/slides/slide"), ".xml")
148	n, err := strconv.Atoi(base)
149	if err != nil {
150		return 1 << 30
151	}
152	return n
153}
154
155// xlsx keeps every string in one shared table and the cells hold indexes into
156// it, so the table has to be read before any sheet means anything.
157func xlsxText(zr *zip.Reader) (string, error) {
158	shared := sharedStrings(zr)
159
160	names := []string{}
161	for _, f := range zr.File {
162		if strings.HasPrefix(f.Name, "xl/worksheets/sheet") && strings.HasSuffix(f.Name, ".xml") {
163			names = append(names, f.Name)
164		}
165	}
166	sort.Strings(names)
167
168	var sb strings.Builder
169	for _, name := range names {
170		data, err := openPart(zr, name)
171		if err != nil {
172			continue
173		}
174		if len(names) > 1 {
175			fmt.Fprintf(&sb, "--- %s ---\n", strings.TrimSuffix(strings.TrimPrefix(name, "xl/worksheets/"), ".xml"))
176		}
177		sb.WriteString(sheetText(data, shared))
178		sb.WriteString("\n")
179	}
180	out := collapseBlankLines(sb.String())
181	if strings.TrimSpace(out) == "" {
182		return "", fmt.Errorf("this xlsx has no values in it")
183	}
184	return out, nil
185}
186
187func sharedStrings(zr *zip.Reader) []string {
188	data, err := openPart(zr, "xl/sharedStrings.xml")
189	if err != nil {
190		return nil
191	}
192	var out []string
193	dec := xml.NewDecoder(bytes.NewReader(data))
194	var cur strings.Builder
195	inItem, inText := false, false
196	for {
197		tok, err := dec.Token()
198		if err != nil {
199			break
200		}
201		switch t := tok.(type) {
202		case xml.StartElement:
203			switch t.Name.Local {
204			case "si":
205				inItem, cur = true, strings.Builder{}
206			case "t":
207				inText = true
208			}
209		case xml.EndElement:
210			switch t.Name.Local {
211			case "si":
212				if inItem {
213					out = append(out, cur.String())
214				}
215				inItem = false
216			case "t":
217				inText = false
218			}
219		case xml.CharData:
220			if inItem && inText {
221				cur.Write(t)
222			}
223		}
224	}
225	return out
226}
227
228// sheetText renders a sheet as tab separated rows, which is the shape a model
229// reads a table in most reliably and is what a csv would have looked like.
230func sheetText(data []byte, shared []string) string {
231	dec := xml.NewDecoder(bytes.NewReader(data))
232	var sb strings.Builder
233	var row []string
234	var cell strings.Builder
235	cellType, inValue := "", false
236
237	for {
238		tok, err := dec.Token()
239		if err != nil {
240			break
241		}
242		switch t := tok.(type) {
243		case xml.StartElement:
244			switch t.Name.Local {
245			case "row":
246				row = row[:0]
247			case "c":
248				cellType, cell = "", strings.Builder{}
249				for _, a := range t.Attr {
250					if a.Name.Local == "t" {
251						cellType = a.Value
252					}
253				}
254			case "v", "t":
255				inValue = true
256			}
257		case xml.EndElement:
258			switch t.Name.Local {
259			case "v", "t":
260				inValue = false
261			case "c":
262				row = append(row, resolveCell(cell.String(), cellType, shared))
263			case "row":
264				line := strings.TrimRight(strings.Join(row, "\t"), "\t")
265				if strings.TrimSpace(line) != "" {
266					sb.WriteString(line)
267					sb.WriteString("\n")
268				}
269			}
270		case xml.CharData:
271			if inValue {
272				cell.Write(t)
273			}
274		}
275	}
276	return sb.String()
277}
278
279// A cell of type s holds an index into the shared table rather than a value,
280// which is the one thing that makes a spreadsheet unreadable if missed.
281func resolveCell(raw, cellType string, shared []string) string {
282	if cellType == "s" {
283		if i, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil && i >= 0 && i < len(shared) {
284			return shared[i]
285		}
286		return ""
287	}
288	return strings.TrimSpace(raw)
289}
290
291// collapseBlankLines keeps a run of empty lines down to one. A docx paragraph
292// per line plus a break element per line doubles them otherwise, and blank
293// lines are tokens the model pays for.
294func collapseBlankLines(s string) string {
295	lines := strings.Split(normalise(s), "\n")
296	out := make([]string, 0, len(lines))
297	blank := false
298	for _, l := range lines {
299		l = strings.TrimRight(l, " \t")
300		if l == "" {
301			if blank {
302				continue
303			}
304			blank = true
305		} else {
306			blank = false
307		}
308		out = append(out, l)
309	}
310	return strings.TrimSpace(strings.Join(out, "\n"))
311}