diff --git a/biz/attachment.go b/biz/attachment.go index cdc7b58..69f505a 100644 --- a/biz/attachment.go +++ b/biz/attachment.go @@ -378,6 +378,8 @@ func (s *AttachmentAPIService) uploadImage(ctx *gin.Context, attachmentType int) if err != nil { ctx.JSON(http.StatusInternalServerError, ginResponse{Code: http.StatusInternalServerError, Message: err.Error(), Error: err.Error()}) } + // 标记删除旧头像附件记录 + s.dbModel.GetDB().Where("type = ? AND type_id = ?", model.AttachmentTypeAvatar, userCliams.UserId).Delete(&model.Attachment{}) } // 保存附件信息 diff --git a/model/article.go b/model/article.go index a669833..7ab7314 100644 --- a/model/article.go +++ b/model/article.go @@ -172,8 +172,7 @@ func (m *DBModel) DeleteArticle(ids []int64) (err error) { return } - // 附件,标记删除:将type_id设置为0 - err = sess.Model(&Attachment{}).Where("type = ? and type_id in (?)", AttachmentTypeArticle, ids).Update("type_id", 0).Error + err = sess.Where("type = ? and type_id in (?)", AttachmentTypeArticle, ids).Delete(&Attachment{}).Error if err != nil { m.logger.Error("DeleteArticle", zap.Error(err)) return diff --git a/model/attachment.go b/model/attachment.go index e72e946..1cacadf 100644 --- a/model/attachment.go +++ b/model/attachment.go @@ -19,7 +19,7 @@ const ( AttachmentTypeComment // 评论 AttachmentTypeBanner // 横幅 AttachmentTypeCategoryCover // 分类封面 - AttachmentTypeConfig // 分类封面 + AttachmentTypeConfig // 配置 ) var attachmentTypeName = map[int]string{ diff --git a/model/banner.go b/model/banner.go index 5779578..360364e 100644 --- a/model/banner.go +++ b/model/banner.go @@ -127,8 +127,7 @@ func (m *DBModel) DeleteBanner(ids []int64) (err error) { return } - // 附件,标记删除 - err = sess.Model(&Attachment{}).Where("type = ? and type_id in (?)", AttachmentTypeBanner, ids).Update("type_id", 0).Error + err = sess.Where("type = ? and type_id in (?)", AttachmentTypeBanner, ids).Delete(&Attachment{}).Error if err != nil { m.logger.Error("DeleteBanner", zap.Error(err)) return diff --git a/model/document.go b/model/document.go index f7bd776..26af020 100644 --- a/model/document.go +++ b/model/document.go @@ -286,8 +286,8 @@ func (m *DBModel) DeleteDocument(ids []int64, deletedUserId int64, deepDelete .. } }() - if len(deepDelete) > 0 && deepDelete[0] { // 彻底删除 - err = sess.Model(&Attachment{}).Where("type = ? and type_id in (?)", AttachmentTypeDocument, ids).Update("type_id", 0).Error + if len(deepDelete) > 0 && deepDelete[0] { // 标记附件为删除状态 + err = sess.Where("type = ? and type_id in (?)", AttachmentTypeDocument, ids).Delete(&Attachment{}).Error if err != nil { m.logger.Error("DeleteDocument", zap.Error(err)) return diff --git a/model/init.go b/model/init.go index 21a5df2..cf2787c 100644 --- a/model/init.go +++ b/model/init.go @@ -132,6 +132,8 @@ func NewDBModel(cfg *conf.Database, lg *zap.Logger) (m *DBModel, err error) { } go m.loopCovertDocument() go m.cronUpdateSitemap() + go m.cronMarkAttachmentDeleted() + go m.cronCleanInvalidAttachment() return } diff --git a/model/util.go b/model/util.go index ebc4349..fb10b78 100644 --- a/model/util.go +++ b/model/util.go @@ -4,6 +4,7 @@ import ( "fmt" "moredoc/util/sitemap" "os" + "path/filepath" "strconv" "strings" "time" @@ -111,7 +112,7 @@ func (m *DBModel) cronUpdateSitemap() { layout := "2006-01-02" lastUpdated := time.Now().Format(layout) for { - hour, _ := strconv.Atoi(os.Getenv("CRON_UPDATE_SITEMAP_HOUR")) // 默认为每天凌晨0点更新站点地图 + hour, _ := strconv.Atoi(os.Getenv("MOREDOC_UPDATE_SITEMAP_HOUR")) // 默认为每天凌晨0点更新站点地图 hour = hour % 24 m.logger.Info("cronUpdateSitemap", zap.Int("hour", hour), zap.String("lastUpdated", lastUpdated)) now := time.Now() @@ -127,3 +128,107 @@ func (m *DBModel) cronUpdateSitemap() { time.Sleep(1 * time.Minute) } } + +// 清理无效附件 +// 1. 找出已被标记删除的附件 +// 2. 查询是否存在相同hash的未被标记删除的附件,对于此类附件,则只删除附件记录而不删除附件文件。 +// 3. 删除已被标记删除的附件 +// 4. 对于文档类附件,要注意衍生的附件,如缩略图、PDF等,也要一并删除。 +func (m *DBModel) cronCleanInvalidAttachment() { + sleepDuration := 1 * time.Minute + for { + time.Sleep(1 * time.Second) + m.logger.Info("cronCleanInvalidAttachment,start...") + var ( + deletedAttachemnts, attachemnts []Attachment + hashes []string + hashMap = make(map[string]struct{}) + ids []int64 + beforeHour, _ = strconv.Atoi(os.Getenv("MOREDOC_CLEAN_ATTACHMENT")) // 默认为每天凌晨0点更新站点地图 + ) + + if beforeHour <= 0 { + beforeHour = 24 + } + + // 1. 找出已被标记删除的附件 + m.db.Unscoped().Where("deleted_at IS NOT NULL").Where("deleted_at < ?", time.Now().Add(-time.Duration(beforeHour)*time.Hour)).Limit(100).Find(&deletedAttachemnts) + if len(deletedAttachemnts) == 0 { + m.logger.Info("cronCleanInvalidAttachment,end...") + time.Sleep(sleepDuration) + continue + } + + for _, attachemnt := range deletedAttachemnts { + hashes = append(hashes, attachemnt.Hash) + ids = append(ids, attachemnt.Id) + } + + // 2. 查询是否存在相同hash的未被标记删除的附件 + m.db.Select("hash").Where("hash IN (?)", hashes).Group("hash").Limit(len(hashes)).Find(&attachemnts) + for _, attachemnt := range attachemnts { + hashMap[attachemnt.Hash] = struct{}{} + } + + // 3. 删除已被标记删除的附件 + err := m.db.Unscoped().Where("id IN (?)", ids).Delete(&Attachment{}).Error + if err != nil { + m.logger.Error("cronCleanInvalidAttachment", zap.Error(err)) + m.logger.Info("cronCleanInvalidAttachment,end...") + continue + } + m.logger.Info("cronCleanInvalidAttachment", zap.Any("ids", ids), zap.Any("Attachemnts", deletedAttachemnts)) + for _, attachemnt := range deletedAttachemnts { + if _, ok := hashMap[attachemnt.Hash]; !ok { // 删除附件文件 + m.logger.Debug("cronCleanInvalidAttachment", zap.String("path", attachemnt.Path), zap.Any("attachemnt", attachemnt)) + file := strings.TrimLeft(attachemnt.Path, "./") + m.logger.Debug("cronCleanInvalidAttachment", zap.String("file", file)) + if err := os.Remove(file); err != nil { + m.logger.Error("cronCleanInvalidAttachment", zap.Error(err), zap.String("file", file)) + } + if attachemnt.Type == AttachmentTypeDocument { // 删除文档的衍生文件 + folder := strings.TrimSuffix(file, filepath.Ext(file)) + m.logger.Debug("cronCleanInvalidAttachment", zap.String("folder", folder)) + if err := os.RemoveAll(folder); err != nil { + m.logger.Error("cronCleanInvalidAttachment", zap.Error(err), zap.String("folder", folder)) + } + } + } + } + m.logger.Info("cronCleanInvalidAttachment,end...") + } +} + +func (m *DBModel) cronMarkAttachmentDeleted() { + // 定时标记删除24小时前上传的但是未被使用的附件 + for { + time.Sleep(1 * time.Hour) + // 1. 查找图片类配置 + var ( + configs []Config + hashes []string + ) + m.db.Select("value").Where("input_type = ?", "image").Find(&configs) + if len(configs) > 0 { + for _, config := range configs { + // 文件hash + hash := strings.TrimSpace(strings.TrimSuffix(filepath.Base(config.Value), filepath.Ext(config.Value))) + if hash != "" { + hashes = append(hashes, hash) + } + } + err := m.db.Where("`hash` NOT IN (?) and `type` = ?", hashes, AttachmentTypeConfig).Delete(&Attachment{}).Error + if err != nil { + m.logger.Error("cronMarkAttachmentDeleted", zap.Error(err)) + } + } + + // 非配置类附件,如果type_id为0,则表示未被使用,超过24小时则标记删除 + m.logger.Info("cronMarkAttachmentDeleted start...") + err := m.db.Where("type != ? and type_id = ?", AttachmentTypeConfig, 0).Where("created_at < ?", time.Now().Add(-time.Duration(24)*time.Hour)).Delete(&Attachment{}).Error + if err != nil { + m.logger.Error("cronMarkAttachmentDeleted", zap.Error(err)) + } + m.logger.Info("cronMarkAttachmentDeleted end...") + } +}