自动清理无效附件

biz/attachment.go

@@ -378,6 +378,8 @@ func (s *AttachmentAPIService) uploadImage(ctx *gin.Context, attachmentType int)
 		if err != nil {
 			ctx.JSON(http.StatusInternalServerError, ginResponse{Code: http.StatusInternalServerError, Message: err.Error(), Error: err.Error()})
 		}
+		// 标记删除旧头像附件记录
+		s.dbModel.GetDB().Where("type = ? AND type_id = ?", model.AttachmentTypeAvatar, userCliams.UserId).Delete(&model.Attachment{})
 	}
 
 	// 保存附件信息

model/article.go

@@ -172,8 +172,7 @@ func (m *DBModel) DeleteArticle(ids []int64) (err error) {
 		return
 	}
 
-	// 附件，标记删除：将type_id设置为0
-	err = sess.Model(&Attachment{}).Where("type = ? and type_id in (?)", AttachmentTypeArticle, ids).Update("type_id", 0).Error
+	err = sess.Where("type = ? and type_id in (?)", AttachmentTypeArticle, ids).Delete(&Attachment{}).Error
 	if err != nil {
 		m.logger.Error("DeleteArticle", zap.Error(err))
 		return

model/attachment.go

@@ -19,7 +19,7 @@ const (
 	AttachmentTypeComment              // 评论
 	AttachmentTypeBanner               // 横幅
 	AttachmentTypeCategoryCover        // 分类封面
-	AttachmentTypeConfig               // 分类封面
+	AttachmentTypeConfig               // 配置
 )
 
 var attachmentTypeName = map[int]string{

model/banner.go

@@ -127,8 +127,7 @@ func (m *DBModel) DeleteBanner(ids []int64) (err error) {
 		return
 	}
 
-	// 附件，标记删除
-	err = sess.Model(&Attachment{}).Where("type = ? and type_id in (?)", AttachmentTypeBanner, ids).Update("type_id", 0).Error
+	err = sess.Where("type = ? and type_id in (?)", AttachmentTypeBanner, ids).Delete(&Attachment{}).Error
 	if err != nil {
 		m.logger.Error("DeleteBanner", zap.Error(err))
 		return

model/document.go

@@ -286,8 +286,8 @@ func (m *DBModel) DeleteDocument(ids []int64, deletedUserId int64, deepDelete ..
 		}
 	}()
 
-	if len(deepDelete) > 0 && deepDelete[0] { // 彻底删除
-		err = sess.Model(&Attachment{}).Where("type = ? and type_id in (?)", AttachmentTypeDocument, ids).Update("type_id", 0).Error
+	if len(deepDelete) > 0 && deepDelete[0] { // 标记附件为删除状态
+		err = sess.Where("type = ? and type_id in (?)", AttachmentTypeDocument, ids).Delete(&Attachment{}).Error
 		if err != nil {
 			m.logger.Error("DeleteDocument", zap.Error(err))
 			return

model/init.go

@@ -132,6 +132,8 @@ func NewDBModel(cfg *conf.Database, lg *zap.Logger) (m *DBModel, err error) {
 	}
 	go m.loopCovertDocument()
 	go m.cronUpdateSitemap()
+	go m.cronMarkAttachmentDeleted()
+	go m.cronCleanInvalidAttachment()
 	return
 }
 

model/util.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"moredoc/util/sitemap"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -111,7 +112,7 @@ func (m *DBModel) cronUpdateSitemap() {
 	layout := "2006-01-02"
 	lastUpdated := time.Now().Format(layout)
 	for {
-		hour, _ := strconv.Atoi(os.Getenv("CRON_UPDATE_SITEMAP_HOUR")) // 默认为每天凌晨0点更新站点地图
+		hour, _ := strconv.Atoi(os.Getenv("MOREDOC_UPDATE_SITEMAP_HOUR")) // 默认为每天凌晨0点更新站点地图
 		hour = hour % 24
 		m.logger.Info("cronUpdateSitemap", zap.Int("hour", hour), zap.String("lastUpdated", lastUpdated))
 		now := time.Now()
@@ -127,3 +128,107 @@ func (m *DBModel) cronUpdateSitemap() {
 		time.Sleep(1 * time.Minute)
 	}
 }
+
+// 清理无效附件
+// 1. 找出已被标记删除的附件
+// 2. 查询是否存在相同hash的未被标记删除的附件，对于此类附件，则只删除附件记录而不删除附件文件。
+// 3. 删除已被标记删除的附件
+// 4. 对于文档类附件，要注意衍生的附件，如缩略图、PDF等，也要一并删除。
+func (m *DBModel) cronCleanInvalidAttachment() {
+	sleepDuration := 1 * time.Minute
+	for {
+		time.Sleep(1 * time.Second)
+		m.logger.Info("cronCleanInvalidAttachment，start...")
+		var (
+			deletedAttachemnts, attachemnts []Attachment
+			hashes                          []string
+			hashMap                         = make(map[string]struct{})
+			ids                             []int64
+			beforeHour, _                   = strconv.Atoi(os.Getenv("MOREDOC_CLEAN_ATTACHMENT")) // 默认为每天凌晨0点更新站点地图
+		)
+
+		if beforeHour <= 0 {
+			beforeHour = 24
+		}
+
+		// 1. 找出已被标记删除的附件
+		m.db.Unscoped().Where("deleted_at IS NOT NULL").Where("deleted_at < ?", time.Now().Add(-time.Duration(beforeHour)*time.Hour)).Limit(100).Find(&deletedAttachemnts)
+		if len(deletedAttachemnts) == 0 {
+			m.logger.Info("cronCleanInvalidAttachment，end...")
+			time.Sleep(sleepDuration)
+			continue
+		}
+
+		for _, attachemnt := range deletedAttachemnts {
+			hashes = append(hashes, attachemnt.Hash)
+			ids = append(ids, attachemnt.Id)
+		}
+
+		// 2. 查询是否存在相同hash的未被标记删除的附件
+		m.db.Select("hash").Where("hash IN (?)", hashes).Group("hash").Limit(len(hashes)).Find(&attachemnts)
+		for _, attachemnt := range attachemnts {
+			hashMap[attachemnt.Hash] = struct{}{}
+		}
+
+		// 3. 删除已被标记删除的附件
+		err := m.db.Unscoped().Where("id IN (?)", ids).Delete(&Attachment{}).Error
+		if err != nil {
+			m.logger.Error("cronCleanInvalidAttachment", zap.Error(err))
+			m.logger.Info("cronCleanInvalidAttachment，end...")
+			continue
+		}
+		m.logger.Info("cronCleanInvalidAttachment", zap.Any("ids", ids), zap.Any("Attachemnts", deletedAttachemnts))
+		for _, attachemnt := range deletedAttachemnts {
+			if _, ok := hashMap[attachemnt.Hash]; !ok { // 删除附件文件
+				m.logger.Debug("cronCleanInvalidAttachment", zap.String("path", attachemnt.Path), zap.Any("attachemnt", attachemnt))
+				file := strings.TrimLeft(attachemnt.Path, "./")
+				m.logger.Debug("cronCleanInvalidAttachment", zap.String("file", file))
+				if err := os.Remove(file); err != nil {
+					m.logger.Error("cronCleanInvalidAttachment", zap.Error(err), zap.String("file", file))
+				}
+				if attachemnt.Type == AttachmentTypeDocument { // 删除文档的衍生文件
+					folder := strings.TrimSuffix(file, filepath.Ext(file))
+					m.logger.Debug("cronCleanInvalidAttachment", zap.String("folder", folder))
+					if err := os.RemoveAll(folder); err != nil {
+						m.logger.Error("cronCleanInvalidAttachment", zap.Error(err), zap.String("folder", folder))
+					}
+				}
+			}
+		}
+		m.logger.Info("cronCleanInvalidAttachment，end...")
+	}
+}
+
+func (m *DBModel) cronMarkAttachmentDeleted() {
+	// 定时标记删除24小时前上传的但是未被使用的附件
+	for {
+		time.Sleep(1 * time.Hour)
+		// 1. 查找图片类配置
+		var (
+			configs []Config
+			hashes  []string
+		)
+		m.db.Select("value").Where("input_type = ?", "image").Find(&configs)
+		if len(configs) > 0 {
+			for _, config := range configs {
+				// 文件hash
+				hash := strings.TrimSpace(strings.TrimSuffix(filepath.Base(config.Value), filepath.Ext(config.Value)))
+				if hash != "" {
+					hashes = append(hashes, hash)
+				}
+			}
+			err := m.db.Where("`hash` NOT IN (?) and `type` = ?", hashes, AttachmentTypeConfig).Delete(&Attachment{}).Error
+			if err != nil {
+				m.logger.Error("cronMarkAttachmentDeleted", zap.Error(err))
+			}
+		}
+
+		// 非配置类附件，如果type_id为0，则表示未被使用，超过24小时则标记删除
+		m.logger.Info("cronMarkAttachmentDeleted start...")
+		err := m.db.Where("type != ?  and type_id = ?", AttachmentTypeConfig, 0).Where("created_at < ?", time.Now().Add(-time.Duration(24)*time.Hour)).Delete(&Attachment{}).Error
+		if err != nil {
+			m.logger.Error("cronMarkAttachmentDeleted", zap.Error(err))
+		}
+		m.logger.Info("cronMarkAttachmentDeleted end...")
+	}
+}