From d1d96e35472f692ace7b08822d185f14913e0ea9 Mon Sep 17 00:00:00 2001 From: clsr Date: Thu, 16 Jun 2016 02:19:44 +0200 Subject: Initial commit --- .gitignore | 5 + api.go | 176 +++++++++++++++++++++++++++++++++++ magic.go | 31 +++++++ main.go | 86 +++++++++++++++++ pomf-standard.txt | 57 ++++++++++++ storage.go | 270 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ website.go | 95 +++++++++++++++++++ 7 files changed, 720 insertions(+) create mode 100644 .gitignore create mode 100644 api.go create mode 100644 magic.go create mode 100644 main.go create mode 100644 pomf-standard.txt create mode 100644 storage.go create mode 100644 website.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..942ae99 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*.swp +/gomf +/pages/ +/static/ +/upload/ diff --git a/api.go b/api.go new file mode 100644 index 0000000..c05c3df --- /dev/null +++ b/api.go @@ -0,0 +1,176 @@ +package main + +import ( + "encoding/base64" + "encoding/csv" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "math/rand" + "mime" + "net/http" + "os" + "path" + "strconv" + "strings" + "time" +) + +func handleGrill(w http.ResponseWriter, r *http.Request) { + grills, err := ioutil.ReadDir("static/grill/") + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + http.Redirect(w, r, "/static/grill/"+grills[rand.Intn(len(grills))].Name(), http.StatusFound) +} + +func handleFile(w http.ResponseWriter, r *http.Request) { + f, hash, size, modtime, err := storage.Get(strings.TrimRight(r.URL.Path, "/")) + if err != nil { + if _, ok := err.(ErrNotFound); ok { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + defer f.Close() + mtype := mime.TypeByExtension(path.Ext(f.Name())) + if strings.Index(mtype, "text/html") == 0 || strings.Index(mtype, "application/xhtml+xml") == 0 { + mtype = "text/plain" + } + w.Header().Set("Content-Type", mtype) + _ = size + //w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) + w.Header().Set("Content-Security-Policy", "default-src 'none'; media-src 'self'") + w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat)) + w.Header().Set("Expires", modtime.UTC().Add(time.Hour*24*30).Format(http.TimeFormat)) + w.Header().Set("Cache-Control", "max-age=2592000") + w.Header().Set("ETag", "\"sha1:"+hash+"\"") + //io.Copy(w, f) + http.ServeContent(w, r, "", modtime, f) +} + +type result struct { + Url string `json:"url"` + Name string `json:"name"` + Hash string `json:"hash"` + Size int64 `json:"size"` +} + +type response struct { + Success bool `json:"success"` + ErrorCode int `json:"errorcode",omitempty` + Description string `json:"description",omitempty` + Files []result `json:"files",omitempty` +} + +func handleUpload(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + output := r.FormValue("output") + + resp := response{Files: []result{}} + + mr, err := r.MultipartReader() + if err != nil { + resp.ErrorCode = http.StatusInternalServerError + resp.Description = err.Error() + respond(w, output, resp) + return + } + + for { + part, err := mr.NextPart() + if err != nil { + if err != io.EOF { + resp.ErrorCode = http.StatusInternalServerError + resp.Description = err.Error() + } + break + } + + if part.FormName() != "files[]" { + continue + } + + id, hash, size, err := storage.New(part, part.FileName()) + if err != nil { + resp.ErrorCode = http.StatusInternalServerError + if _, ok := err.(ErrTooLarge); ok { + resp.ErrorCode = http.StatusRequestEntityTooLarge + } else if _, ok := err.(ErrForbidden); ok { + resp.ErrorCode = http.StatusForbidden + } + break + } + + bhash, _ := base64.RawURLEncoding.DecodeString(hash) + res := result{ + Name: part.FileName(), + Url: strings.TrimRight(uploadUrl, "/") + "/" + id, + Hash: hex.EncodeToString(bhash), + Size: size, + } + resp.Files = append(resp.Files, res) + + part.Close() + } + + respond(w, output, resp) +} + +func respond(w http.ResponseWriter, mode string, resp response) { + if resp.ErrorCode != 0 { + resp.Files = []result{} + resp.Success = false + } else { + resp.Success = true + } + + code := http.StatusOK + if resp.ErrorCode != 0 { + code = resp.ErrorCode + } + w.WriteHeader(code) + + switch mode { + case "json": + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(resp) + + case "text", "gyazo": + w.Header().Set("Content-Type", "text/plain") + sep := "" + for _, file := range resp.Files { + io.WriteString(w, sep+file.Url) + sep = "\n" + } + if mode != "gyazo" { + io.WriteString(w, "\n") + } + + case "csv": + w.Header().Set("Content-Type", "text/csv") + wr := csv.NewWriter(w) + wr.Write([]string{"name", "url", "hash", "size"}) + for _, file := range resp.Files { + wr.Write([]string{file.Name, file.Url, file.Hash, strconv.FormatInt(file.Size, 10)}) + } + wr.Flush() + + case "", "html": + w.Header().Set("Content-Type", "text/html") + context := newContext() + context.Result = resp + if err := templates.ExecuteTemplate(w, "index.html", context); err != nil { + fmt.Fprintln(os.Stderr, err) + } + + default: + respond(w, "", response{ErrorCode: http.StatusNotFound, Description: "invalid output mode " + mode}) + return + } +} diff --git a/magic.go b/magic.go new file mode 100644 index 0000000..c93e15d --- /dev/null +++ b/magic.go @@ -0,0 +1,31 @@ +package main + +// #cgo LDFLAGS: -lmagic +// #include +// #include +import "C" +import "errors" +import "unsafe" + +var magic C.magic_t + +func init() { + magic = C.magic_open(C.MAGIC_MIME_TYPE | C.MAGIC_SYMLINK | C.MAGIC_ERROR) + if magic == nil { + panic("unable to initialize libmagic") + } + if C.magic_load(magic, nil) != 0 { + C.magic_close(magic) + panic("unable to load libmagic database: " + C.GoString(C.magic_error(magic))) + } +} + +func GetMimeType(fname string) (string, error) { + cfname := C.CString(fname) + defer C.free(unsafe.Pointer(cfname)) + mime := C.magic_file(magic, cfname) + if mime == nil { + return "", errors.New(C.GoString(C.magic_error(magic))) + } + return C.GoString(mime), nil +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..fb8810f --- /dev/null +++ b/main.go @@ -0,0 +1,86 @@ +package main + +import ( + "flag" + "fmt" + "math/rand" + "net/http" + "strings" + "time" +) + +var storage *Storage + +var ( + uploadUrl string + uploadHost string + siteName string + contactMail string + abuseMail string + hsts bool +) + +func handle(w http.ResponseWriter, r *http.Request) { + if hsts { + w.Header().Set("Strict-Transport-Security", "max-age=15552000") + } + if uploadHost != "" && r.URL.Host == uploadHost { + handleFile(w, r) + } else { + http.DefaultServeMux.ServeHTTP(w, r) + } +} + +func main() { + flag.StringVar(&uploadUrl, "upload-url", "", "URL to serve uploads from") + flag.StringVar(&uploadHost, "upload-host", "", "host to serve uploads on") + flag.StringVar(&siteName, "name", "Gomf", "website name") + flag.StringVar(&contactMail, "contact", "contact@example.com", "contact email address") + flag.StringVar(&abuseMail, "abuse", "abuse@example.com", "abuse email address") + flag.BoolVar(&hsts, "hsts", false, "enable HSTS") + listenHttp := flag.String("http", "localhost:8080", "address to listen on for HTTP") + listenHttps := flag.String("https", "", "address to listen on for HTTPS") + cert := flag.String("cert", "", "path to TLS certificate (for HTTPS)") + key := flag.String("key", "", "path to TLS key (for HTTPS)") + maxSize := flag.Int64("max-size", 50*1024*1024, "max filesize in bytes") + forbidMime := flag.String("forbid-mime", "application/x-dosexec,application/x-msdos-program", "comma-separated list of forbidden MIME types") + forbidExt := flag.String("forbid-ext", "exe,dll,msi,scr,com,pif", "comma-separated list of forbidden file extensions") + flag.Parse() + + rand.Seed(time.Now().UnixNano()) + + storage = NewStorage("upload", *maxSize) + storage.ForbiddenExt = strings.Split(*forbidExt, ",") + storage.ForbiddenMime = strings.Split(*forbidMime, ",") + + http.HandleFunc("/upload.php", handleUpload) + http.HandleFunc("/grill.php", handleGrill) + http.Handle("/u/", http.StripPrefix("/u/", http.HandlerFunc(handleFile))) + + initWebsite() + + if uploadUrl == "" { + if *listenHttps != "" { + uploadUrl = "https://" + *listenHttps + "/u/" + } else if *listenHttp != "" { + uploadUrl = "http://" + *listenHttp + "/u/" + } + } + + exit := true + if *listenHttp != "" { + exit = false + fmt.Printf("listening on http://%s/\n", *listenHttp) + go panic(http.ListenAndServe(*listenHttp, http.HandlerFunc(handle))) + } + if *listenHttps != "" { + exit = false + fmt.Printf("listening on https://%s/\n", *listenHttps) + go panic(http.ListenAndServeTLS(*listenHttps, *cert, *key, http.HandlerFunc(handle))) + } + + if !exit { + switch { + } + } +} diff --git a/pomf-standard.txt b/pomf-standard.txt new file mode 100644 index 0000000..f623d87 --- /dev/null +++ b/pomf-standard.txt @@ -0,0 +1,57 @@ +Upload API endpoint: + /upload.php + + +POST arguments: + files[]: + Content-Type: multipart/form-data + File to upload; multiple values supported. + + +GET arguments: + output: + gyazo: + Content-Type: text/plain + Complete URLs to uploaded files in the same order as the input files, separated by newlines. Does not have a trailing newline (Pomf1 compat version). + Example output: 'https://example.com/foobar.jpg\nhttps://example.com/qweasd.txt' + + text: + Content-Type: text/plain + Complete URLs to uploaded files in the same order as input files. Each line ends in a newline (Unix style). + Example output: 'https://example.com/foobar.jpg\nhttps://example.com/qweasd.txt\n' + Protip: if you include input names of uploaded files, how do you handle e.g. newlines in filenames? + + html: + Content-Type: text/html + A HTML page containing links to uploaded files. Can be anything and is primarily meant to be shown to a human user. + Example output: 'https://example.com/foobar.jpg
https://example.com/qweasd.txt
' + + json: + Content-Type: application/json + Schema: + { + "success": bool /* true if everything is okay, false if there was an error */, + "errorcode": int /* only if success=false, the HTTP error code */, + "description": string /* only if success=false, the error message */, + "files": [ + { + "name": string /* original filename sent by the client */, + "url": string /* the complete URL to the uploaded file */, + "hash": string /* the SHA-1 hash of the uploaded file */, + "size": int /* the bytesize of the uploaded file */ + } + ] /* only if success=true, info about uploaded files in the same order they were uploaded */ + } + Clients *must not* assume a specific ordering of keys in objects nor any presence/absence of whitespace (outside strings); regex is not a good way to parse this. + Example output: '{"success": true, "files": [{"name": "cat.jpg", "url": "https://example.com/foobar.jpg", "hash": "8d26e24aabb26c02b5c9a9e102308af2a3597a49", "size": 44294}, {"name": "file.txt", "url": "https://example.com/qweasd.txt", "hash": "da39a3ee5e6b4b0d3255bfef95601890afd80709", "size": 0}]}' + + csv: + Content-Type: text/csv + A CSV document listing the name, url, hash and size of uploaded files (same meanings as in the JSON response). + Dialect: delimiter=',', quotechar='"' + Headers are written on the first line. + Example output: 'name,url,hash,size\ncat.jpg,https://example.com/foobar.jpg,8d26e24aabb26c02b5c9a9e102308af2a3597a49,44294\nfile.txt,https://example.com/qweasd.txt,da39a3ee5e6b4b0d3255bfef95601890afd80709,0\n' + + +Rationale: + Such an API would provide the maximum compatibility with Pomf1 and Pomf2 while still implementing all the important features. diff --git a/storage.go b/storage.go new file mode 100644 index 0000000..588b121 --- /dev/null +++ b/storage.go @@ -0,0 +1,270 @@ +package main + +import ( + "crypto/sha1" + "encoding/base64" + "encoding/hex" + "errors" + "io" + "io/ioutil" + "math/rand" + "mime" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "time" +) + +const ( + MaxIdTries = 64 + + DefaultIdCharset = "abcdefghijklmnopqrstuvwxyz" + DefaultIdLength = 6 + DefaultMaxSize = 50 * 1024 * 1024 +) + +type Storage struct { + Folder string + IdCharset string + IdLength int + MaxSize int64 + ForbiddenMime []string + ForbiddenExt []string +} + +type ErrForbidden struct{ Type string } + +func (e ErrForbidden) Error() string { return "forbidden type: " + e.Type } + +type ErrTooLarge struct{ Size int64 } + +func (e ErrTooLarge) Error() string { + return "file exceeds maximum allowed size of " + strconv.FormatInt(e.Size, 10) + " bytes" +} + +type ErrNotFound struct{ Name string } + +func (e ErrNotFound) Error() string { return "file " + e.Name + " not found" } + +func NewStorage(folder string, maxSize int64) *Storage { + if err := os.MkdirAll(path.Join(folder, "temp"), 0755); err != nil { + panic(err) + } + if err := os.MkdirAll(path.Join(folder, "files"), 0755); err != nil { + panic(err) + } + if err := os.MkdirAll(path.Join(folder, "ids"), 0755); err != nil { + panic(err) + } + + return &Storage{ + Folder: folder, + IdCharset: DefaultIdCharset, + IdLength: DefaultIdLength, + MaxSize: maxSize, + } +} + +func (s *Storage) Get(id string) (file *os.File, hash string, size int64, modtime time.Time, err error) { + ext := path.Ext(id) + id = id[:len(id)-len(ext)] + for i := 0; i < len(id); i++ { + if !strings.ContainsRune(s.IdCharset, rune(id[i])) { + err = errors.New("invalid ID") + } + } + folder := s.idToFolder("ids", id) + files, err := ioutil.ReadDir(folder) + if err != nil { + err = ErrNotFound{id + ext} + return + } + if len(files) < 1 { + err = errors.New("internal storage error") + return + } + fn := files[0].Name() + fp := path.Join(folder, fn) + target, err := os.Readlink(fp) + if err != nil { + return + } + bhash, _ := base64.RawStdEncoding.DecodeString(path.Base(path.Dir(target))) + hash = hex.EncodeToString(bhash) + if path.Ext(fn) != ext { + err = ErrNotFound{id + ext} + return + } + stat, err := os.Lstat(fp) + if err != nil { + return + } + modtime = stat.ModTime() + size = stat.Size() + file, err = os.Open(fp) + return +} + +var errFileExists = errors.New("file exists") + +func (s *Storage) New(r io.Reader, name string) (id, hash string, size int64, err error) { + temp, err := ioutil.TempFile(path.Join(s.Folder, "temp"), "file") + if err != nil { + return + } + defer func() { + if temp != nil { + temp.Close() + os.Remove(temp.Name()) + } + }() + + hash, size, err = s.readInput(temp, r) + if err != nil { + return + } + _, ext, err := s.getMimeExt(temp.Name(), name) + if err != nil { + return + } + id, err = s.storeFile(temp, hash, ext) + if err == nil { + temp = nil // prevent deletion + } else if err == errFileExists { + err = nil + } + + return +} + +func (s *Storage) randomId() string { + id := make([]byte, s.IdLength) + for i := 0; i < len(id); i++ { + id[i] = s.IdCharset[rand.Intn(len(s.IdCharset))] + } + return string(id) +} + +func (s *Storage) idToFolder(subfolder, id string) string { + for len(id) < 4 { + id = "_" + id + } + return path.Join(s.Folder, subfolder, id[0:1], id[1:3], id) +} + +func (s *Storage) readInput(w io.Writer, r io.Reader) (hash string, size int64, err error) { + h := sha1.New() + w = io.MultiWriter(h, w) + if s.MaxSize > 0 { + r = io.LimitReader(r, s.MaxSize+1) + } + size, err = io.Copy(w, r) + if err != nil { + return + } + if lr, ok := r.(*io.LimitedReader); ok && lr.N == 0 { + err = ErrTooLarge{s.MaxSize} + return + } + hash = base64.RawURLEncoding.EncodeToString(h.Sum(nil)) + return +} + +func (s *Storage) getMimeExt(fpath string, name string) (mimetype, ext string, err error) { + mimetype, err = GetMimeType(fpath) + if err != nil { + return + } + + // choose file extension, prefer the user-provided one + ext = path.Ext(name) + exts, err := mime.ExtensionsByType(mimetype) + valid := false + if err != nil { + return + } + if ext != "" { + for _, e := range exts { + if e == ext { + valid = true + break + } + } + } + if !valid { + ext = "" + if len(exts) > 0 { + ext = exts[0] + } + } + + // reject forbidden MIME types and file extensions + if mimetype != "application/octet-stream" { + for _, e := range exts { + for _, fe := range s.ForbiddenExt { + if e == fe { + err = ErrForbidden{fe} + return + } + } + } + for _, fm := range s.ForbiddenMime { + if mimetype == fm { + err = ErrForbidden{fm} + return + } + } + } + + return +} + +func (s *Storage) storeFile(file *os.File, hash, ext string) (id string, err error) { + hfolder := s.idToFolder("files", hash) + hpath := path.Join(hfolder, "file") + fexists := false + + os.MkdirAll(path.Dir(hfolder), 0755) + err = os.Mkdir(hfolder, 0755) + if err != nil { + if _, err = os.Stat(hpath); err != nil { + err = errors.New("internal storage error") + } + fexists = true + } else { + err = os.Rename(file.Name(), hpath) + os.Chmod(hpath, 0644) + } + if err != nil { + return + } + + fpath := "" + for i := 0; i < MaxIdTries; i++ { + id = s.randomId() + dir := s.idToFolder("ids", id) + os.MkdirAll(path.Dir(dir), 0755) + err = os.Mkdir(dir, 0755) + if err == nil { + fpath = path.Join(dir, "file"+ext) + id += ext + break + } + } + if fpath == "" { + err = errors.New("internal storage error") + return + } + rhpath, err := filepath.Rel(path.Dir(fpath), hpath) + if err != nil { + return + } + err = os.Symlink(rhpath, fpath) + + if fexists && err == nil { + err = errFileExists + } + return +} diff --git a/website.go b/website.go new file mode 100644 index 0000000..e601bcd --- /dev/null +++ b/website.go @@ -0,0 +1,95 @@ +package main + +import ( + "fmt" + "html/template" + "io/ioutil" + "net/http" + "os" + "path" + "strconv" + "strings" +) + +var templates *template.Template + +func initWebsite() { + os.MkdirAll("pages", 0755) + pages, err := ioutil.ReadDir("pages") + if err != nil { + panic(err) + } + + templates = template.Must(template.ParseGlob("pages/*.html")) + + for _, page := range pages { + if path.Ext(page.Name()) == ".html" { + http.HandleFunc("/"+page.Name(), handlePage) + if page.Name() == "index.html" { + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.NotFound(w, r) + return + } + handlePage(w, r) + }) + } + } + } + + http.Handle("/static/", http.StripPrefix("/static", http.FileServer(http.Dir("static")))) + http.HandleFunc("/favicon.ico", handleFavicon) +} + +func humanize(bytes int64) string { + units := []string{"B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"} + i := 0 + n := float64(bytes) + for n >= 1024 && i < len(units)-1 { + n /= 1024 + i += 1 + } + return strconv.FormatFloat(n, 'f', -1, 64) + " " + units[i] +} + +func handleFavicon(w http.ResponseWriter, r *http.Request) { + http.ServeFile(w, r, "static/favicon.ico") +} + +type pageContext struct { + SiteName string + Abuse string + Contact string + MaxSizeBytes int64 + MaxSize string + Pages map[string]string + Result response +} + +func newContext() pageContext { + pages := make(map[string]string) + for _, t := range templates.Templates() { + n := t.Name() + title := n[:len(n)-len(path.Ext(n))] + title = strings.ToUpper(title[0:1]) + title[1:] + pages[title] = n + } + return pageContext{ + SiteName: siteName, + Abuse: abuseMail, + Contact: contactMail, + MaxSizeBytes: storage.MaxSize, + MaxSize: humanize(storage.MaxSize), + Pages: pages, + } +} + +func handlePage(w http.ResponseWriter, r *http.Request) { + page := strings.TrimLeft(r.URL.Path, "/") + if page == "" { + page = "index.html" + } + if err := templates.ExecuteTemplate(w, page, newContext()); err != nil { + fmt.Fprintln(os.Stderr, err) + } +} -- cgit