Skip to content

Commit cb9acf5

Browse files
FabianKrammFabianKramm
authored
Merge pull request #331 from covexo/issue-326
Issue 326
2 parents ac65d49 + de27851 commit cb9acf5

2 files changed

Lines changed: 134 additions & 137 deletions

File tree

‎pkg/devspace/sync/downstream.go‎

Lines changed: 133 additions & 127 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,6 @@ func (d *downstream) mainLoop() error {
109109
}
110110

111111
amountChanges := len(createFiles) + len(removeFiles)
112-
113112
if lastAmountChanges > 0 && amountChanges == lastAmountChanges {
114113
err = d.applyChanges(createFiles, removeFiles)
115114
if err != nil {
@@ -150,6 +149,139 @@ func (d *downstream) cloneFileMap() map[string]*fileInformation {
150149
return mapClone
151150
}
152151

152+
func (d *downstream) collectChanges(removeFiles map[string]*fileInformation) ([]*fileInformation, error) {
153+
createFiles := make([]*fileInformation, 0, 128)
154+
destPathFound := false
155+
156+
// Write find command to stdin pipe
157+
cmd := getFindCommand(d.config.DestPath)
158+
_, err := d.stdinPipe.Write([]byte(cmd))
159+
if err != nil {
160+
return nil, errors.Trace(err)
161+
}
162+
163+
buf := make([]byte, 0, 512)
164+
overlap := ""
165+
done := false
166+
167+
for done == false {
168+
n, err := d.stdoutPipe.Read(buf[:cap(buf)])
169+
buf = buf[:n]
170+
171+
if n == 0 {
172+
if err == nil {
173+
continue
174+
}
175+
if err == io.EOF {
176+
return nil, errors.Trace(fmt.Errorf("[Downstream] Stream closed unexpectedly"))
177+
}
178+
179+
return nil, errors.Trace(err)
180+
}
181+
182+
// Error reading from stdout
183+
if err != nil && err != io.EOF {
184+
return nil, errors.Trace(err)
185+
}
186+
187+
done, overlap, err = d.parseLines(string(buf), overlap, &createFiles, removeFiles, &destPathFound)
188+
if err != nil {
189+
if _, ok := err.(parsingError); ok {
190+
time.Sleep(time.Second * 4)
191+
return d.collectChanges(removeFiles)
192+
}
193+
194+
// No trace here because it could be a parsing error
195+
return nil, errors.Trace(err)
196+
}
197+
}
198+
199+
if destPathFound == false {
200+
return nil, errors.New("DestPath not found, find command did not execute correctly")
201+
}
202+
203+
return createFiles, nil
204+
}
205+
206+
func (d *downstream) parseLines(buffer, overlap string, createFiles *[]*fileInformation, removeFiles map[string]*fileInformation, destPathFound *bool) (bool, string, error) {
207+
lines := strings.Split(buffer, "\n")
208+
209+
for index, element := range lines {
210+
line := ""
211+
212+
if index == 0 {
213+
if len(lines) > 1 {
214+
line = overlap + element
215+
} else {
216+
overlap += element
217+
}
218+
} else if index == len(lines)-1 {
219+
overlap = element
220+
} else {
221+
line = element
222+
}
223+
224+
if line == EndAck || overlap == EndAck {
225+
return true, overlap, nil
226+
} else if line == ErrorAck || overlap == ErrorAck {
227+
return true, "", parsingError{
228+
msg: "Parsing Error",
229+
}
230+
} else if line != "" {
231+
destPath, err := d.evaluateFile(line, createFiles, removeFiles)
232+
if destPath {
233+
*destPathFound = destPath
234+
}
235+
236+
if err != nil {
237+
return true, "", errors.Trace(err)
238+
}
239+
}
240+
}
241+
242+
return false, overlap, nil
243+
}
244+
245+
func (d *downstream) evaluateFile(fileline string, createFiles *[]*fileInformation, removeFiles map[string]*fileInformation) (bool, error) {
246+
d.config.fileIndex.fileMapMutex.Lock()
247+
defer d.config.fileIndex.fileMapMutex.Unlock()
248+
249+
fileInformation, err := parseFileInformation(fileline, d.config.DestPath)
250+
251+
// Error parsing line
252+
if err != nil {
253+
return false, errors.Trace(err)
254+
}
255+
256+
// No file found
257+
if fileInformation == nil {
258+
return true, nil
259+
}
260+
261+
// File found don't delete it
262+
delete(removeFiles, fileInformation.Name)
263+
264+
// Update mode, gid & uid if exists
265+
if d.config.fileIndex.fileMap[fileInformation.Name] != nil {
266+
d.config.fileIndex.fileMap[fileInformation.Name].RemoteMode = fileInformation.RemoteMode
267+
d.config.fileIndex.fileMap[fileInformation.Name].RemoteGID = fileInformation.RemoteGID
268+
d.config.fileIndex.fileMap[fileInformation.Name].RemoteUID = fileInformation.RemoteUID
269+
}
270+
271+
// Exclude symlinks
272+
if fileInformation.IsSymbolicLink {
273+
// Add them to the fileMap though
274+
d.config.fileIndex.fileMap[fileInformation.Name] = fileInformation
275+
}
276+
277+
// Should we download the file / folder?
278+
if shouldDownload(fileInformation, d.config) {
279+
*createFiles = append(*createFiles, fileInformation)
280+
}
281+
282+
return false, nil
283+
}
284+
153285
func (d *downstream) applyChanges(createFiles []*fileInformation, removeFiles map[string]*fileInformation) error {
154286
var err error
155287

@@ -384,129 +516,3 @@ func (d *downstream) createFolders(createFolders []*fileInformation) {
384516
}
385517
}
386518
}
387-
388-
func (d *downstream) collectChanges(removeFiles map[string]*fileInformation) ([]*fileInformation, error) {
389-
createFiles := make([]*fileInformation, 0, 128)
390-
391-
// Write find command to stdin pipe
392-
cmd := getFindCommand(d.config.DestPath)
393-
_, err := d.stdinPipe.Write([]byte(cmd))
394-
if err != nil {
395-
return nil, errors.Trace(err)
396-
}
397-
398-
buf := make([]byte, 0, 512)
399-
overlap := ""
400-
done := false
401-
402-
for done == false {
403-
n, err := d.stdoutPipe.Read(buf[:cap(buf)])
404-
buf = buf[:n]
405-
406-
if n == 0 {
407-
if err == nil {
408-
continue
409-
}
410-
411-
if err == io.EOF {
412-
return nil, errors.Trace(fmt.Errorf("[Downstream] Stream closed unexpectedly"))
413-
}
414-
415-
return nil, errors.Trace(err)
416-
}
417-
418-
// Error reading from stdout
419-
if err != nil && err != io.EOF {
420-
return nil, errors.Trace(err)
421-
}
422-
423-
done, overlap, err = d.parseLines(string(buf), overlap, &createFiles, removeFiles)
424-
if err != nil {
425-
if _, ok := err.(parsingError); ok {
426-
time.Sleep(time.Second * 4)
427-
return d.collectChanges(removeFiles)
428-
}
429-
430-
// No trace here because it could be a parsing error
431-
return nil, errors.Trace(err)
432-
}
433-
}
434-
435-
return createFiles, nil
436-
}
437-
438-
func (d *downstream) parseLines(buffer, overlap string, createFiles *[]*fileInformation, removeFiles map[string]*fileInformation) (bool, string, error) {
439-
lines := strings.Split(buffer, "\n")
440-
441-
for index, element := range lines {
442-
line := ""
443-
444-
if index == 0 {
445-
if len(lines) > 1 {
446-
line = overlap + element
447-
} else {
448-
overlap += element
449-
}
450-
} else if index == len(lines)-1 {
451-
overlap = element
452-
} else {
453-
line = element
454-
}
455-
456-
if line == EndAck || overlap == EndAck {
457-
return true, overlap, nil
458-
} else if line == ErrorAck || overlap == ErrorAck {
459-
return true, "", parsingError{
460-
msg: "Parsing Error",
461-
}
462-
} else if line != "" {
463-
err := d.evaluateFile(line, createFiles, removeFiles)
464-
465-
if err != nil {
466-
return true, "", errors.Trace(err)
467-
}
468-
}
469-
}
470-
471-
return false, overlap, nil
472-
}
473-
474-
func (d *downstream) evaluateFile(fileline string, createFiles *[]*fileInformation, removeFiles map[string]*fileInformation) error {
475-
d.config.fileIndex.fileMapMutex.Lock()
476-
defer d.config.fileIndex.fileMapMutex.Unlock()
477-
478-
fileInformation, err := parseFileInformation(fileline, d.config.DestPath)
479-
480-
// Error parsing line
481-
if err != nil {
482-
return errors.Trace(err)
483-
}
484-
485-
// No file found
486-
if fileInformation == nil {
487-
return nil
488-
}
489-
490-
// File found don't delete it
491-
delete(removeFiles, fileInformation.Name)
492-
493-
// Update mode, gid & uid if exists
494-
if d.config.fileIndex.fileMap[fileInformation.Name] != nil {
495-
d.config.fileIndex.fileMap[fileInformation.Name].RemoteMode = fileInformation.RemoteMode
496-
d.config.fileIndex.fileMap[fileInformation.Name].RemoteGID = fileInformation.RemoteGID
497-
d.config.fileIndex.fileMap[fileInformation.Name].RemoteUID = fileInformation.RemoteUID
498-
}
499-
500-
// Exclude symlinks
501-
if fileInformation.IsSymbolicLink {
502-
// Add them to the fileMap though
503-
d.config.fileIndex.fileMap[fileInformation.Name] = fileInformation
504-
}
505-
506-
// Should we download the file / folder?
507-
if shouldDownload(fileInformation, d.config) {
508-
*createFiles = append(*createFiles, fileInformation)
509-
}
510-
511-
return nil
512-
}

‎pkg/devspace/sync/file_information.go‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -56,48 +56,42 @@ func (p parsingError) Error() string {
5656
}
5757

5858
func getFindCommand(destPath string) string {
59-
return "mkdir -p '" + destPath + "' && find -L '" + destPath + "' -exec stat -c \"%n///%s,%Y,%f,%a,%u,%g\" {} + 2>/dev/null && echo -n \"" + EndAck + "\" || echo \"" + ErrorAck + "\"\n"
59+
return "mkdir -p '" + destPath + "' && find -L '" + destPath + "' -exec stat -c \"%n///%s,%Y,%f,%a,%u,%g\" {} + 2>/dev/null && echo -n \"" + EndAck + "\" || echo -n \"" + ErrorAck + "\"\n"
6060
}
6161

6262
func parseFileInformation(fileline, destPath string) (*fileInformation, error) {
6363
fileinfo := fileInformation{}
6464

6565
t := strings.Split(fileline, "///")
66-
6766
if len(t) != 2 {
6867
return nil, errors.New("[Downstream] Wrong fileline: " + fileline)
6968
}
70-
7169
if len(t[0]) <= len(destPath) {
7270
return nil, nil
7371
}
7472

7573
fileinfo.Name = t[0][len(destPath):]
7674

7775
t = strings.Split(t[1], ",")
78-
7976
if len(t) != 6 {
8077
return nil, errors.New("[Downstream] Wrong fileline: " + fileline)
8178
}
8279

8380
size, err := strconv.Atoi(t[0])
84-
8581
if err != nil {
8682
return nil, errors.Trace(err)
8783
}
8884

8985
fileinfo.Size = int64(size)
9086

9187
mTime, err := strconv.Atoi(t[1])
92-
9388
if err != nil {
9489
return nil, errors.Trace(err)
9590
}
9691

9792
fileinfo.Mtime = int64(mTime)
9893

9994
rawMode, err := strconv.ParseUint(t[2], 16, 32) // Parse hex string into uint64
100-
10195
if err != nil {
10296
return nil, errors.Trace(err)
10397
}
@@ -107,23 +101,20 @@ func parseFileInformation(fileline, destPath string) (*fileInformation, error) {
107101
fileinfo.IsDirectory = (rawMode & IsDirectory) == IsDirectory
108102

109103
mode, err := strconv.ParseInt(t[3], 8, 32)
110-
111104
if err != nil {
112105
return nil, errors.Trace(err)
113106
}
114107

115108
fileinfo.RemoteMode = mode
116109

117110
uid, err := strconv.Atoi(t[4])
118-
119111
if err != nil {
120112
return nil, errors.Trace(err)
121113
}
122114

123115
fileinfo.RemoteUID = uid
124116

125117
gid, err := strconv.Atoi(t[5])
126-
127118
if err != nil {
128119
return nil, errors.Trace(err)
129120
}

0 commit comments

Comments
 (0)