-
Notifications
You must be signed in to change notification settings - Fork 1
/
core.go
157 lines (136 loc) · 3.46 KB
/
core.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
package fsdd
import (
"runtime"
"sync"
"time"
)
type file struct {
name string // full qualified file name (path/name)
size uint64 // data size in bytes
inode uint64 // inode number
hash [32]byte // file hash
fasthash uint64 // file fast hash
symlink bool // true if file is a symlink
syminvalid bool // true if file is a symlink & invalid
newSymLink bool // true if new symlink is requested
symtarget string // pre-existing symlink target
newlinktarget string // target filename (hard|sym)
}
var (
// global config
c *Config
// fs state analysis
xfs = make(map[uint64][]string) // map inode to []name(s)
nfs = make(map[uint64][]uint64) // map size to inode
sfs = make(map[string]uint64) // map name to size
ifs = make(map[string]uint64) // map name to inode
hfs = make(map[[32]byte][]string) // map hash to name
hFs = make(map[uint64][]string) // map fasthash to name
sym = make(map[string]string) // map existing symlinks
syi []string // array of invalid symlinks
// global channel
fileChan = make(chan *file, 10000)
hashChan = make(chan *file, 10000)
linkChan = make(chan *file, 10000)
failChan = make(chan *file, 10000)
metaChan = make(chan string, 10000)
dirChan = make(chan string, 10000)
namesChan = make(chan string, 10000)
rmsymChan = make(chan string, 10000)
// global locks
bg, ctl, global sync.WaitGroup
// global counter
total, hsave, hcount, stotal, sinodes, ssym, failsize, failinodes, failssym, failisym uint64
)
func (config *Config) run() {
// init options (r/o) via global
c = optionsSanityCheck(config)
// tick
t0 := time.Now()
reportInit()
// config
worker := runtime.NumCPU()
// recursive tree walk
{
// spin up collector
global.Add(1)
go collectTreeWalker()
// spin up recursive file tree walker
if c.CleanMetadata {
ctl.Add(worker)
go cleanMetadata(worker)
}
go workerTreeWalker(worker)
// init treeWalk
bg.Add(1)
dirChan <- c.Path
// wait for recursive treewalk
bg.Wait()
close(dirChan)
close(fileChan)
close(metaChan)
ctl.Wait()
}
global.Wait()
// file hashing
{
// spin up hashsum worker collector
global.Add(1)
go collectFileHash()
// spin up hashsum worker
bg.Add(worker)
go workerFileHash(worker)
// feed same-size-files into hashsum worker
go feedFileHash()
// wait
bg.Wait()
close(hashChan)
}
global.Wait()
// apply changes
{
// collect stats
global.Add(1)
go collectStats()
// spin up file system change worker
switch {
case c.HardLink || c.SymLink || c.ReplaceSymlinks:
bg.Add(worker)
go workerLink(worker)
case c.RemoveBrokenSymlinks:
bg.Add(worker)
go workerRemoveBrokenSymlinks(worker)
}
// feed identified dupes & clean targets into linker/remove/stats channel
ctl.Add(1)
go feedLinkStats()
if c.ReplaceSymlinks {
ctl.Add(1)
go feederReplaceSymlinks()
}
if c.RemoveBrokenSymlinks {
ctl.Add(1)
go feederRemoveBrokenSymlinks()
}
// wait
ctl.Wait()
close(rmsymChan)
close(linkChan)
bg.Wait()
close(failChan)
}
global.Wait()
// report summary when finished
reportSummary(t0)
}
//
// LITTLE HELPER
//
// optionsSanityCheck ...
func optionsSanityCheck(config *Config) *Config {
config.Path = verifyPath(pathSanitizer(config.Path))
if config.HardLink && config.SymLink {
errExit("[hardlink and symlink activated] [please choose only one target method]")
}
return config
}