1 package server
2
3 import (
4 "archive/zip"
5 "database/sql"
6 "encoding/base64"
7 "fmt"
8 "html"
9 "html/template"
10 "io"
11 "net/http"
12 "os"
13 "path/filepath"
14 "regexp"
15 "strconv"
16 "strings"
17
18 "codeberg.org/tslocum/sriracha/internal/database"
19 . "codeberg.org/tslocum/sriracha/model"
20 . "codeberg.org/tslocum/sriracha/util"
21 "github.com/gabriel-vasile/mimetype"
22 )
23
24 var ytEmbedPattern = regexp.MustCompile(`\/\/www\.youtube\.com\/embed\/([0-9A-Za-z_\-]+)`)
25
26 type importInfo struct {
27 name string
28 sqlDB *sql.DB
29 posts []*Post
30 }
31
32 func (s *Server) _importDatabase(name string, filePath string) error {
33 sqlDB, err := sql.Open("sqlite", filePath)
34 if err != nil {
35 return fmt.Errorf("failed to open file %s: expected SQLite database file", filePath)
36 }
37 info := &importInfo{
38 name: name,
39 sqlDB: sqlDB,
40 }
41 s.importDatabases = append(s.importDatabases, info)
42 return nil
43 }
44
45 func (s *Server) importDatabase(importPath string) error {
46 _, err := os.Stat(importPath)
47 if os.IsNotExist(err) {
48 return fmt.Errorf("file %s does not exist", importPath)
49 }
50 z, err := zip.OpenReader(importPath)
51 if err != nil {
52 return s._importDatabase(filepath.Base(importPath), importPath)
53 }
54 for _, f := range z.File {
55 tmpFile, err := os.CreateTemp("", "*.db")
56 if err != nil {
57 return fmt.Errorf("failed to create temporary file: %s", err)
58 }
59 zf, err := f.Open()
60 if err != nil {
61 return fmt.Errorf("failed to open file %s in archive: %s", f.Name, err)
62 }
63 _, err = io.Copy(tmpFile, zf)
64 if err != nil {
65 return fmt.Errorf("failed to extract file %s from archive: %s", f.Name, err)
66 }
67 err = s._importDatabase(f.Name, tmpFile.Name())
68 if err != nil {
69 return fmt.Errorf("failed to import file %s: %s", f.Name, err)
70 }
71 }
72 return nil
73 }
74
75 func (s *Server) _importPost(p *Post, tinyIB bool) error {
76 if p.Board == nil {
77 return nil
78 }
79
80
81 if tinyIB && (p.FileHash == "YouTube" || p.FileHash == "Vimeo" || p.FileHash == "SoundCloud") {
82 ytVideo := p.FileHash == "YouTube"
83
84
85 p.FileHash = "e " + p.FileHash + " " + p.FileOriginal
86 p.FileOriginal = ""
87
88
89 if ytVideo {
90 m := ytEmbedPattern.FindStringSubmatch(p.File)
91 if len(m) > 1 {
92 p.FileOriginal = "https://www.youtube.com/watch?v=" + m[1]
93 }
94 }
95 }
96
97
98 if p.Bumped <= 0 {
99 p.Bumped = p.Timestamp
100 }
101
102
103 if p.NameBlock == "" {
104 p.SetNameBlock(p.Board.DefaultName, "", false)
105 }
106
107 if p.File != "" && !p.IsEmbed() {
108 srcPath := filepath.Join(s.config.Root, p.Board.Dir, "src", p.File)
109
110 srcFile, err := os.Open(srcPath)
111 if err != nil {
112 return fmt.Errorf("failed to open attachment %s of post No.%d: %s", p.File, p.ID, err)
113 }
114 defer srcFile.Close()
115
116
117 if p.FileMIME == "" {
118 mime, err := mimetype.DetectReader(srcFile)
119 if err == nil {
120 p.FileMIME = mime.String()
121 }
122 }
123
124
125 hash := s.newHash()
126 srcFile.Seek(0, 0)
127 fileSize, err := io.Copy(hash, srcFile)
128 if err != nil {
129 return fmt.Errorf("failed to calculate hash of attachment %s of post No.%d: %s", p.File, p.ID, err)
130 }
131 var sum [HashSize]byte
132 hash.Sum(sum[:0])
133 p.FileHash = base64.URLEncoding.EncodeToString(sum[:])
134 p.FileSize = fileSize
135
136
137 if p.FileWidth <= 0 || p.FileHeight <= 0 {
138 isImage := p.FileMIME == "image/jpeg" || p.FileMIME == "image/pjpeg" || p.FileMIME == "image/png" || p.FileMIME == "image/gif"
139 if isImage {
140 srcFile.Seek(0, 0)
141 imgWidth, imgHeight := s.imageDimensions(srcFile)
142 if imgWidth == 0 || imgHeight == 0 {
143 return fmt.Errorf("failed to calculate width and height of attachment %s of post No.%d: %s", p.File, p.ID, err)
144 }
145 p.FileWidth, p.FileHeight = imgWidth, imgHeight
146 }
147 }
148 }
149 if p.Thumb != "" {
150 thumbPath := filepath.Join(s.config.Root, p.Board.Dir, "thumb", p.Thumb)
151
152 thumbFile, err := os.Open(thumbPath)
153 if err != nil {
154 return fmt.Errorf("failed to open attachment %s of post No.%d: %s", p.File, p.ID, err)
155 }
156 defer thumbFile.Close()
157
158
159 if p.ThumbWidth <= 0 || p.ThumbHeight <= 0 {
160 mime, err := mimetype.DetectReader(thumbFile)
161 if err == nil {
162 mimeType := mime.String()
163 if mimeType == "image/jpeg" || mimeType == "image/pjpeg" || mimeType == "image/png" || mimeType == "image/gif" {
164 imgWidth, imgHeight := s.imageDimensions(thumbFile)
165 if imgWidth != 0 && imgHeight != 0 {
166 p.ThumbWidth, p.ThumbHeight = imgWidth, imgHeight
167 }
168 }
169 }
170 if p.ThumbWidth <= 0 || p.ThumbHeight <= 0 {
171 p.Thumb = ""
172 p.ThumbWidth = 0
173 p.ThumbHeight = 0
174 }
175 }
176 }
177 return nil
178 }
179
180 func (s *Server) importPosts(sqlDB *sql.DB, table string, tinyIB bool, b *Board) ([]*Post, error) {
181
182 var query string
183 if tinyIB {
184
185 query = "SELECT id, parent, timestamp, bumped, name, tripcode, email, nameblock, subject, message, file, '' AS file_mime, file_hex, file_original, file_size, image_width, image_height, thumb, thumb_width, thumb_height, stickied, locked FROM " + table
186 } else {
187
188 query = "SELECT id, parent, timestamp, bumped, name, tripcode, email, nameblock, subject, message, file, filemime, filehash, fileoriginal, filesize, filewidth, fileheight, thumb, thumbwidth, thumbheight, stickied, locked FROM " + table
189 }
190 query += " ORDER BY id ASC"
191
192
193 var posts []*Post
194 rows, err := sqlDB.Query(query)
195 if err != nil {
196 return nil, err
197 }
198 for rows.Next() {
199 p := &Post{}
200 var stickied, locked int
201 err = rows.Scan(&p.ID,
202 &p.Parent,
203 &p.Timestamp,
204 &p.Bumped,
205 &p.Name,
206 &p.Tripcode,
207 &p.Email,
208 &p.NameBlock,
209 &p.Subject,
210 &p.Message,
211 &p.File,
212 &p.FileMIME,
213 &p.FileHash,
214 &p.FileOriginal,
215 &p.FileSize,
216 &p.FileWidth,
217 &p.FileHeight,
218 &p.Thumb,
219 &p.ThumbWidth,
220 &p.ThumbHeight,
221 &stickied,
222 &locked)
223 if err != nil {
224 return nil, err
225 }
226 p.Moderated = ModeratedVisible
227 p.Stickied = stickied == 1
228 p.Locked = locked == 1
229
230 posts = append(posts, p)
231
232 if b == nil {
233 continue
234 }
235 p.Board = b
236 err = s._importPost(p, tinyIB)
237 if err != nil {
238 return nil, err
239 }
240 }
241 return posts, nil
242 }
243
244 func (s *Server) serveImport(data *templateData, db *database.DB, w http.ResponseWriter, r *http.Request) {
245 data.Template = "manage_info"
246 data.Boards = db.AllBoards()
247 data.Message = `<h2 class="managetitle">` + GetHTML(nil, data.Account, "Import") + `</h2>`
248
249 completeMessage := "<b>" + Get(nil, data.Account, "Import complete.") + "</b><br>" + Get(nil, data.Account, "Please restart Sriracha without the %s flag.", "--import") + "<br>"
250 if data.forbidden(w, RoleSuperAdmin) {
251 return
252 } else if !s.config.ImportMode {
253 data.ManageError(Get(nil, data.Account, "Sriracha is not running in import mode."))
254 return
255 } else if s.config.ImportComplete {
256 data.Message += template.HTML(completeMessage)
257 return
258 }
259
260 data.Template = "manage_info"
261 data.Message += `<b>` + GetHTML(nil, data.Account, "Warning") + `:</b> ` + GetHTML(nil, data.Account, "Backup all files and databases before importing posts.") + `<br><br>`
262
263 commit := FormBool(r, "import") && FormBool(r, "confirm")
264 defer func() {
265 if commit && s.config.ImportComplete {
266 err := db.CommitWithErr()
267 if err != nil {
268 data.ManageError("Failed to commit changes: " + err.Error())
269 return
270 } else {
271 data.Message += template.HTML(completeMessage)
272 }
273 } else {
274 db.RollBack()
275 }
276 }()
277
278 var haveMapping bool
279 importBoards := make([]*Board, len(s.importDatabases))
280 for i := range s.importDatabases {
281 boardID := FormInt(r, fmt.Sprintf("board%d", i))
282 if boardID > 0 {
283 importBoards[i] = db.BoardByID(boardID)
284 if importBoards[i] != nil {
285 haveMapping = true
286 }
287 }
288 }
289
290
291 for i, info := range s.importDatabases {
292 sqlDB := info.sqlDB
293
294
295 var table string
296 var tinyIB bool
297 for i := 0; i < 2; i++ {
298 column := "filesize"
299 if i == 1 {
300 column = "file_size"
301 }
302 rows, err := sqlDB.Query("SELECT DISTINCT name FROM sqlite_master WHERE sql LIKE '%" + column + "%'")
303 if err != nil {
304 data.ManageError(err.Error())
305 return
306 }
307 for rows.Next() {
308 err = rows.Scan(&table)
309 if err != nil {
310 data.ManageError(err.Error())
311 return
312 }
313 }
314 if table != "" {
315 tinyIB = i == 1
316 break
317 }
318 }
319 if table == "" {
320 data.ManageError(fmt.Sprintf("Failed to locate post table in export %s", info.name))
321 return
322 }
323
324 posts, err := s.importPosts(sqlDB, table, tinyIB, importBoards[i])
325 if err != nil {
326 data.ManageError(fmt.Sprintf("Failed to load export %s: %s", info.name, err.Error()))
327 return
328 } else if len(posts) == 0 {
329 data.ManageError(fmt.Sprintf("No posts were found in export %s.", info.name))
330 return
331 }
332 s.importDatabases[i].posts = posts
333 }
334
335 if !haveMapping {
336 data.Message += `<table class="managetable"><tbody>
337 <tr>
338 <th>` + GetHTML(nil, data.Account, "Export") + `</th>
339 <th>` + GetHTML(nil, data.Account, "Threads") + `</th>
340 <th>` + GetHTML(nil, data.Account, "Replies") + `</th>
341 <th>` + GetHTML(nil, data.Account, "Posts") + `</th>
342 </tr>`
343 for _, info := range s.importDatabases {
344 name := strings.TrimSuffix(strings.TrimSuffix(info.name, ".db"), ".sriracha")
345 var threads, replies int
346 for _, p := range info.posts {
347 if p.Parent == 0 {
348 threads++
349 } else {
350 replies++
351 }
352 }
353 data.Message += template.HTML(s.msgPrinter.Sprintf("<tr><td>%s</td><td>%d</td><td>%d</td><td>%d</td></tr>", html.EscapeString(name), threads, replies, threads+replies))
354 }
355 data.Message += `</tbody></table>`
356 }
357
358 if !haveMapping || !commit {
359 if !haveMapping {
360 data.Message += template.HTML("<br><b>" + Get(nil, data.Account, "Export files loaded.") + "</b><br>" + Get(nil, data.Account, "Ready to start dry run.") + "<br>")
361 } else if !commit {
362 data.Message += template.HTML("<b>" + Get(nil, data.Account, "Dry run complete.") + "</b><br>" + Get(nil, data.Account, "Ready to import posts.") + "<br>")
363 }
364
365 data.Message += template.HTML(`<br><fieldset>
366 <legend>` + Get(nil, data.Account, "Boards") + `</legend>
367 <form method="post">
368 <input type="hidden" name="import" value="1">`)
369 if !haveMapping {
370 data.Message += template.HTML(Get(nil, data.Account, "Choose where to import posts") + `:<br><br>`)
371 } else {
372 data.Message += template.HTML(`<input type="hidden" name="confirm" value="1">`)
373 }
374 data.Message += template.HTML(`<table class="manageform">`)
375 var disabled string
376 if haveMapping {
377 disabled = " disabled"
378 }
379 var selected string
380 for i, info := range s.importDatabases {
381 if haveMapping && importBoards[i] == nil {
382 continue
383 }
384 name := strings.TrimSuffix(strings.TrimSuffix(info.name, ".db"), ".sriracha")
385 data.Message += template.HTML(fmt.Sprintf(`<tr>
386 <td class="postblock"><label for="board%d">%s</label></td>
387 <td><select name="board%d"%s>
388 <option value="0">`+Get(nil, data.Account, "Do not import")+`</option>`, i, name, i, disabled))
389 for _, b := range data.Boards {
390 label := b.Path()
391 if b.Name != "" {
392 label += " " + b.Name
393 }
394 selected = ""
395 if importBoards[i] != nil && b.ID == importBoards[i].ID {
396 selected = " selected"
397 }
398 data.Message += template.HTML(fmt.Sprintf(`<option value="%d"%s>%s</option>`, b.ID, selected, label))
399 }
400 data.Message += template.HTML(`</select></td>
401 </tr>`)
402 }
403 data.Message += template.HTML(`
404 <tr>
405 <td style="vertical-align: middle;"> `)
406 if !haveMapping {
407 data.Message += template.HTML(`[<a href="/sriracha/board/">` + Get(nil, data.Account, "Manage Boards") + `</a>]`)
408 }
409 label := "Start Dry Run"
410 if haveMapping {
411 label = "Start Import"
412 }
413 data.Message += template.HTML(`</td>
414 <td><input type="submit" value="` + Get(nil, data.Account, label) + `"></td>
415 <td></td>
416 </tr>
417 </table>`)
418 if haveMapping {
419 for i := range s.importDatabases {
420 b := importBoards[i]
421 if b != nil {
422 data.Message += template.HTML(fmt.Sprintf(`<input type="hidden" name="board%d" value="%d">`, i, b.ID))
423 }
424 }
425 }
426 data.Message += template.HTML(`</form>
427 </fieldset><br>`)
428 return
429 }
430
431 var lastPostID int
432 for i, info := range s.importDatabases {
433 b := importBoards[i]
434 if b == nil {
435 continue
436 }
437
438 var rewriteIDs bool
439 for _, p := range info.posts {
440 if p.ID <= 0 {
441 data.ManageError(fmt.Sprintf("Invalid post: no post ID: %+v", *p))
442 return
443 }
444 dbPost := db.PostByID(p.ID)
445 if dbPost != nil {
446 rewriteIDs = true
447 break
448 }
449 }
450
451 newIDs := make(map[int]int)
452 for _, p := range info.posts {
453 carriageReturn := regexp.MustCompile(`(?s)\r.?`)
454 p.Message = carriageReturn.ReplaceAllStringFunc(p.Message, func(s string) string {
455 if len(s) == 1 || s[1] == '\n' {
456 return "\n"
457 }
458 return "\n" + string(s[1])
459 })
460
461 resPattern := regexp.MustCompile(`<a href="[^"]*res\/([0-9]+).html#([0-9]+)" class="([A-Aa-z]+)">>>([0-9]+)</a>`)
462 p.Message = resPattern.ReplaceAllStringFunc(p.Message, func(s string) string {
463 match := resPattern.FindStringSubmatch(s)
464 threadID := ParseInt(match[1])
465 postID := ParseInt(match[2])
466 if newIDs[threadID] == 0 || newIDs[postID] == 0 {
467 return s
468 }
469 return fmt.Sprintf(`<a href="%sres/%d.html#%d" class="%s">>>%d</a>`, b.Path(), newIDs[threadID], newIDs[postID], match[3], newIDs[postID])
470 })
471
472 p.Message = strings.TrimSuffix(p.Message, "<br>")
473
474 if p.Parent != 0 {
475 p.Parent = newIDs[p.Parent]
476 }
477 oldID := p.ID
478 if rewriteIDs {
479 db.AddPost(p)
480 } else {
481 var parent *int
482 if p.Parent != 0 {
483 parent = &p.Parent
484 }
485 var fileHash *string
486 if p.FileHash != "" {
487 fileHash = &p.FileHash
488 }
489 var stickied int
490 if p.Stickied {
491 stickied = 1
492 }
493 var locked int
494 if p.Locked {
495 locked = 1
496 }
497 _, err := db.Exec("INSERT INTO post VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26)",
498 p.ID,
499 parent,
500 p.Board.ID,
501 p.Timestamp,
502 p.Bumped,
503 p.IP,
504 p.Name,
505 p.Tripcode,
506 p.Email,
507 p.NameBlock,
508 p.Subject,
509 p.Message,
510 p.Password,
511 p.File,
512 fileHash,
513 p.FileOriginal,
514 p.FileSize,
515 p.FileWidth,
516 p.FileHeight,
517 p.Thumb,
518 p.ThumbWidth,
519 p.ThumbHeight,
520 p.Moderated,
521 stickied,
522 locked,
523 p.FileMIME,
524 )
525 if err != nil {
526 data.ManageError(fmt.Sprintf("Failed to insert post: %s.", err))
527 return
528 }
529 }
530 newIDs[oldID] = p.ID
531 if p.ID > lastPostID {
532 lastPostID = p.ID
533 }
534 }
535
536 if rewriteIDs {
537 _, err := db.Exec("ALTER SEQUENCE post_id_seq RESTART WITH " + strconv.Itoa(db.MaxPostID()+1))
538 if err != nil {
539 data.ManageError(fmt.Sprintf("Failed to update post auto-increment value: %s.", err))
540 return
541 }
542 }
543 }
544
545 s.config.ImportComplete = true
546 s.rebuildAll(db, false)
547 }
548
View as plain text