fedi/forgejo
1
0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo.git synced 2024-11-10 01:05:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove cache avatar support and add its tests

Browse source
This commit is contained in:
Unknwon 2016-02-14 23:14:55 -05:00
parent fd92d91da3
commit 58e004f7da
14 changed files with 45 additions and 390 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.bra.toml
View file

@ -11,6 +11,7 @@ watch_dirs = [
"$WORKDIR/routers"
]
watch_exts = [".go"]
ignore_files = [".+_test.go"]
build_delay = 1500
cmds = [
["make", "build-dev"], # sqlite cert pam tidb

10
Makefile
View file

@ -19,13 +19,13 @@ build: $(GENERATED)
go install -v -ldflags '$(LDFLAGS)' -tags '$(TAGS)'
cp '$(GOPATH)/bin/gogs' .
build-dev: $(GENERATED)
go install -v -race -tags '$(TAGS)'
cp '$(GOPATH)/bin/gogs' .
govet:
go tool vet -composites=false -methods=false -structtags=false .
build-dev: $(GENERATED) govet
go install -v -race -tags '$(TAGS)'
cp '$(GOPATH)/bin/gogs' .
pack:
rm -rf $(RELEASE_GOGS)
mkdir -p $(RELEASE_GOGS)
@ -52,4 +52,4 @@ clean-mac: clean
find . -name ".DS_Store" -print0 | xargs -0 rm
test:
go test ./...
go test -cover -race ./...

2
README.md
View file

@ -3,7 +3,7 @@ Gogs - Go Git Service [![Build Status](https://travis-ci.org/gogits/gogs.svg?bra
![](https://github.com/gogits/gogs/blob/master/public/img/gogs-large-resize.png?raw=true)
##### Current version: 0.8.35
##### Current version: 0.8.36
| Web | UI | Preview |
|:-------------:|:-------:|:-------:|

6
cmd/web.go
View file

@ -34,7 +34,6 @@ import (
"github.com/gogits/gogs/models"
"github.com/gogits/gogs/modules/auth"
"github.com/gogits/gogs/modules/avatar"
"github.com/gogits/gogs/modules/bindata"
"github.com/gogits/gogs/modules/log"
"github.com/gogits/gogs/modules/middleware"
@ -245,11 +244,6 @@ func runWeb(ctx *cli.Context) {
})
// ***** END: User *****
// Gravatar service.
avt := avatar.CacheServer("public/img/avatar/", "public/img/avatar_default.jpg")
os.MkdirAll("public/img/avatar/", os.ModePerm)
m.Get("/avatar/:hash", avt.ServeHTTP)
adminReq := middleware.Toggle(&middleware.ToggleOptions{SignInRequire: true, AdminRequire: true})
// ***** START: Admin *****

2
conf/app.ini
View file

@ -120,8 +120,6 @@ REGISTER_EMAIL_CONFIRM = false
DISABLE_REGISTRATION = false
; User must sign in to view anything.
REQUIRE_SIGNIN_VIEW = false
; Cache avatar as picture
ENABLE_CACHE_AVATAR = false
; Mail notification
ENABLE_NOTIFY_MAIL = false
; More detail: https://github.com/gogits/gogs/issues/165

2
gogs.go
View file

@ -17,7 +17,7 @@ import (
"github.com/gogits/gogs/modules/setting"
)
const APP_VER = "0.8.35.0214"
const APP_VER = "0.8.36.0214"
func init() {
runtime.GOMAXPROCS(runtime.NumCPU())

6
models/user.go
View file

@ -247,8 +247,6 @@ func (u *User) RelAvatarLink() string {
}
return "/avatars/" + com.ToStr(u.Id)
case setting.Service.EnableCacheAvatar:
return "/avatar/" + u.Avatar
}
return setting.GravatarSource + u.Avatar
}
@ -493,7 +491,7 @@ func CreateUser(u *User) (err error) {
u.LowerName = strings.ToLower(u.Name)
u.AvatarEmail = u.Email
u.Avatar = avatar.HashEmail(u.AvatarEmail)
u.Avatar = base.HashEmail(u.AvatarEmail)
u.Rands = GetUserSalt()
u.Salt = GetUserSalt()
u.EncodePasswd()
@ -628,7 +626,7 @@ func updateUser(e Engine, u *User) error {
if len(u.AvatarEmail) == 0 {
u.AvatarEmail = u.Email
}
u.Avatar = avatar.HashEmail(u.AvatarEmail)
u.Avatar = base.HashEmail(u.AvatarEmail)
}
u.LowerName = strings.ToLower(u.Name)

314
modules/avatar/avatar.go
View file

@ -2,74 +2,21 @@
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// for www.gravatar.com image cache
/*
It is recommend to use this way
cacheDir := "./cache"
defaultImg := "./default.jpg"
http.Handle("/avatar/", avatar.CacheServer(cacheDir, defaultImg))
*/
package avatar
import (
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
"image"
"image/color/palette"
"image/jpeg"
"image/png"
"io"
"math/rand"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/issue9/identicon"
"github.com/nfnt/resize"
"github.com/gogits/gogs/modules/log"
"github.com/gogits/gogs/modules/setting"
)
//FIXME: remove cache module
var gravatarSource string
func UpdateGravatarSource() {
gravatarSource = setting.GravatarSource
if strings.HasPrefix(gravatarSource, "//") {
gravatarSource = "http:" + gravatarSource
} else if !strings.HasPrefix(gravatarSource, "http://") &&
!strings.HasPrefix(gravatarSource, "https://") {
gravatarSource = "http://" + gravatarSource
}
log.Debug("avatar.UpdateGravatarSource(update gavatar source): %s", gravatarSource)
}
// hash email to md5 string
// keep this func in order to make this package independent
func HashEmail(email string) string {
// https://en.gravatar.com/site/implement/hash/
email = strings.TrimSpace(email)
email = strings.ToLower(email)
h := md5.New()
h.Write([]byte(email))
return hex.EncodeToString(h.Sum(nil))
}
const _RANDOM_AVATAR_SIZE = 200
// RandomImage generates and returns a random avatar image.
func RandomImage(data []byte) (image.Image, error) {
func RandomImageSize(size int, data []byte) (image.Image, error) {
randExtent := len(palette.WebSafe) - 32
rand.Seed(time.Now().UnixNano())
colorIndex := rand.Intn(randExtent)
@ -78,262 +25,17 @@ func RandomImage(data []byte) (image.Image, error) {
backColorIndex = randExtent - 1
}
// Size, background, forecolor
imgMaker, err := identicon.New(_RANDOM_AVATAR_SIZE,
// Define size, background, and forecolor
imgMaker, err := identicon.New(size,
palette.WebSafe[backColorIndex], palette.WebSafe[colorIndex:colorIndex+32]...)
if err != nil {
return nil, err
return nil, fmt.Errorf("identicon.New: %v", err)
}
return imgMaker.Make(data), nil
}
// Avatar represents the avatar object.
type Avatar struct {
Hash string
AlterImage string // image path
cacheDir string // image save dir
reqParams string
imagePath string
expireDuration time.Duration
}
func New(hash string, cacheDir string) *Avatar {
return &Avatar{
Hash: hash,
cacheDir: cacheDir,
expireDuration: time.Minute * 10,
reqParams: url.Values{
"d": {"retro"},
"size": {"290"},
"r": {"pg"}}.Encode(),
imagePath: filepath.Join(cacheDir, hash+".image"), //maybe png or jpeg
}
}
func (this *Avatar) HasCache() bool {
fileInfo, err := os.Stat(this.imagePath)
return err == nil && fileInfo.Mode().IsRegular()
}
func (this *Avatar) Modtime() (modtime time.Time, err error) {
fileInfo, err := os.Stat(this.imagePath)
if err != nil {
return
}
return fileInfo.ModTime(), nil
}
func (this *Avatar) Expired() bool {
modtime, err := this.Modtime()
return err != nil || time.Since(modtime) > this.expireDuration
}
// default image format: jpeg
func (this *Avatar) Encode(wr io.Writer, size int) (err error) {
var img image.Image
decodeImageFile := func(file string) (img image.Image, err error) {
fd, err := os.Open(file)
if err != nil {
return
}
defer fd.Close()
if img, err = jpeg.Decode(fd); err != nil {
fd.Seek(0, os.SEEK_SET)
img, err = png.Decode(fd)
}
return
}
imgPath := this.imagePath
if !this.HasCache() {
if this.AlterImage == "" {
return errors.New("request image failed, and no alt image offered")
}
imgPath = this.AlterImage
}
if img, err = decodeImageFile(imgPath); err != nil {
return
}
m := resize.Resize(uint(size), 0, img, resize.Lanczos3)
return jpeg.Encode(wr, m, nil)
}
// get image from gravatar.com
func (this *Avatar) Update() {
UpdateGravatarSource()
thunder.Fetch(gravatarSource+this.Hash+"?"+this.reqParams,
this.imagePath)
}
func (this *Avatar) UpdateTimeout(timeout time.Duration) (err error) {
UpdateGravatarSource()
select {
case <-time.After(timeout):
err = fmt.Errorf("get gravatar image %s timeout", this.Hash)
case err = <-thunder.GoFetch(gravatarSource+this.Hash+"?"+this.reqParams,
this.imagePath):
}
return err
}
type service struct {
cacheDir string
altImage string
}
func (this *service) mustInt(r *http.Request, defaultValue int, keys ...string) (v int) {
for _, k := range keys {
if _, err := fmt.Sscanf(r.FormValue(k), "%d", &v); err == nil {
defaultValue = v
}
}
return defaultValue
}
func (this *service) ServeHTTP(w http.ResponseWriter, r *http.Request) {
urlPath := r.URL.Path
hash := urlPath[strings.LastIndex(urlPath, "/")+1:]
size := this.mustInt(r, 290, "s", "size") // default size = 290*290
avatar := New(hash, this.cacheDir)
avatar.AlterImage = this.altImage
if avatar.Expired() {
if err := avatar.UpdateTimeout(time.Millisecond * 1000); err != nil {
log.Trace("avatar update error: %v", err)
return
}
}
if modtime, err := avatar.Modtime(); err == nil {
etag := fmt.Sprintf("size(%d)", size)
if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modtime.Before(t.Add(1*time.Second)) && etag == r.Header.Get("If-None-Match") {
h := w.Header()
delete(h, "Content-Type")
delete(h, "Content-Length")
w.WriteHeader(http.StatusNotModified)
return
}
w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat))
w.Header().Set("ETag", etag)
}
w.Header().Set("Content-Type", "image/jpeg")
if err := avatar.Encode(w, size); err != nil {
log.Warn("avatar encode error: %v", err)
w.WriteHeader(500)
}
}
// http.Handle("/avatar/", avatar.CacheServer("./cache"))
func CacheServer(cacheDir string, defaultImgPath string) http.Handler {
return &service{
cacheDir: cacheDir,
altImage: defaultImgPath,
}
}
// thunder downloader
var thunder = &Thunder{QueueSize: 10}
type Thunder struct {
QueueSize int // download queue size
q chan *thunderTask
once sync.Once
}
func (t *Thunder) init() {
if t.QueueSize < 1 {
t.QueueSize = 1
}
t.q = make(chan *thunderTask, t.QueueSize)
for i := 0; i < t.QueueSize; i++ {
go func() {
for {
task := <-t.q
task.Fetch()
}
}()
}
}
func (t *Thunder) Fetch(url string, saveFile string) error {
t.once.Do(t.init)
task := &thunderTask{
Url: url,
SaveFile: saveFile,
}
task.Add(1)
t.q <- task
task.Wait()
return task.err
}
func (t *Thunder) GoFetch(url, saveFile string) chan error {
c := make(chan error)
go func() {
c <- t.Fetch(url, saveFile)
}()
return c
}
// thunder download
type thunderTask struct {
Url string
SaveFile string
sync.WaitGroup
err error
}
func (this *thunderTask) Fetch() {
this.err = this.fetch()
this.Done()
}
var client = &http.Client{}
func (this *thunderTask) fetch() error {
log.Debug("avatar.fetch(fetch new avatar): %s", this.Url)
req, _ := http.NewRequest("GET", this.Url, nil)
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/jpeg,image/png,*/*;q=0.8")
req.Header.Set("Accept-Encoding", "deflate,sdch")
req.Header.Set("Accept-Language", "zh-CN,zh;q=0.8")
req.Header.Set("Cache-Control", "no-cache")
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.154 Safari/537.36")
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return fmt.Errorf("status code: %d", resp.StatusCode)
}
/*
log.Println("headers:", resp.Header)
switch resp.Header.Get("Content-Type") {
case "image/jpeg":
this.SaveFile += ".jpeg"
case "image/png":
this.SaveFile += ".png"
}
*/
/*
imgType := resp.Header.Get("Content-Type")
if imgType != "image/jpeg" && imgType != "image/png" {
return errors.New("not png or jpeg")
}
*/
tmpFile := this.SaveFile + ".part" // mv to destination when finished
fd, err := os.Create(tmpFile)
if err != nil {
return err
}
_, err = io.Copy(fd, resp.Body)
fd.Close()
if err != nil {
os.Remove(tmpFile)
return err
}
return os.Rename(tmpFile, this.SaveFile)
// RandomImage generates and returns a random avatar image.
// The data should normally be the []byte type of email address.
func RandomImage(data []byte) (image.Image, error) {
return RandomImageSize(_RANDOM_AVATAR_SIZE, data)
}

64
modules/avatar/avatar_test.go
View file

@ -1,61 +1,23 @@
// Copyright 2014 The Gogs Authors. All rights reserved.
// Copyright 2016 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package avatar_test
package avatar
import (
"errors"
"os"
"strconv"
"testing"
"time"
"github.com/gogits/gogs/modules/avatar"
"github.com/gogits/gogs/modules/log"
. "github.com/smartystreets/goconvey/convey"
)
const TMPDIR = "test-avatar"
func Test_RandomImage(t *testing.T) {
Convey("Generate a random avatar from email", t, func() {
_, err := RandomImage([]byte("gogs@local"))
So(err, ShouldBeNil)
func TestFetch(t *testing.T) {
os.Mkdir(TMPDIR, 0755)
defer os.RemoveAll(TMPDIR)
hash := avatar.HashEmail("ssx205@gmail.com")
a := avatar.New(hash, TMPDIR)
a.UpdateTimeout(time.Millisecond * 200)
}
func TestFetchMany(t *testing.T) {
os.Mkdir(TMPDIR, 0755)
defer os.RemoveAll(TMPDIR)
t.Log("start")
var n = 5
ch := make(chan bool, n)
for i := 0; i < n; i++ {
go func(i int) {
hash := avatar.HashEmail(strconv.Itoa(i) + "ssx205@gmail.com")
a := avatar.New(hash, TMPDIR)
a.Update()
t.Log("finish", hash)
ch <- true
}(i)
}
for i := 0; i < n; i++ {
<-ch
}
t.Log("end")
}
// cat
// wget http://www.artsjournal.com/artfulmanager/wp/wp-content/uploads/2013/12/200x200xmirror_cat.jpg.pagespeed.ic.GOZSv6v1_H.jpg -O default.jpg
/*
func TestHttp(t *testing.T) {
http.Handle("/", avatar.CacheServer("./", "default.jpg"))
http.ListenAndServe(":8001", nil)
}
*/
func TestLogTrace(t *testing.T) {
log.Trace("%v", errors.New("console log test"))
Convey("Try to generate an image with size zero", func() {
_, err := RandomImageSize(0, []byte("gogs@local"))
So(err, ShouldNotBeNil)
})
})
}

18
modules/base/tool.go
View file

@ -26,7 +26,6 @@ import (
"github.com/gogits/chardet"
"github.com/gogits/gogs/modules/avatar"
"github.com/gogits/gogs/modules/log"
"github.com/gogits/gogs/modules/setting"
)
@ -209,17 +208,22 @@ func CreateTimeLimitCode(data string, minutes int, startInf interface{}) string
return code
}
// AvatarLink returns avatar link by given e-mail.
// HashEmail hashes email address to MD5 string.
// https://en.gravatar.com/site/implement/hash/
func HashEmail(email string) string {
email = strings.ToLower(strings.TrimSpace(email))
h := md5.New()
h.Write([]byte(email))
return hex.EncodeToString(h.Sum(nil))
}
// AvatarLink returns avatar link by given email.
func AvatarLink(email string) string {
if setting.DisableGravatar || setting.OfflineMode {
return setting.AppSubUrl + "/img/avatar_default.jpg"
}
gravatarHash := avatar.HashEmail(email)
if setting.Service.EnableCacheAvatar {
return setting.AppSubUrl + "/avatar/" + gravatarHash
}
return setting.GravatarSource + gravatarHash
return setting.GravatarSource + HashEmail(email)
}
// Seconds-based time units

4
modules/bindata/bindata.go
View file

File diff suppressed because one or more lines are too long

2
modules/setting/setting.go
View file

@ -455,7 +455,6 @@ var Service struct {
DisableRegistration bool
ShowRegistrationButton bool
RequireSignInView bool
EnableCacheAvatar bool
EnableNotifyMail bool
EnableReverseProxyAuth bool
EnableReverseProxyAutoRegister bool
@ -469,7 +468,6 @@ func newService() {
Service.DisableRegistration = sec.Key("DISABLE_REGISTRATION").MustBool()
Service.ShowRegistrationButton = sec.Key("SHOW_REGISTRATION_BUTTON").MustBool(!Service.DisableRegistration)
Service.RequireSignInView = sec.Key("REQUIRE_SIGNIN_VIEW").MustBool()
Service.EnableCacheAvatar = sec.Key("ENABLE_CACHE_AVATAR").MustBool()
Service.EnableReverseProxyAuth = sec.Key("ENABLE_REVERSE_PROXY_AUTHENTICATION").MustBool()
Service.EnableReverseProxyAutoRegister = sec.Key("ENABLE_REVERSE_PROXY_AUTO_REGISTRATION").MustBool()
Service.EnableCaptcha = sec.Key("ENABLE_CAPTCHA").MustBool()

2
modules/ssh/ssh.go
View file

@ -145,8 +145,6 @@ func listen(config *ssh.ServerConfig, port int) {
go handleServerConn(sConn.Permissions.Extensions["key-id"], chans)
}()
}
panic("SSH: Unexpected exit of listen loop")
}
// Listen starts a SSH server listens on given port.

2
templates/.VERSION
View file

@ -1 +1 @@
0.8.35.0214
0.8.36.0214