Skip to content

Commit bf8f7b4

Browse files
cuixqcuixq
authored
refactor(vulnfeeds): switch to osv-schema Go bindings in cmd/ids (#4933)
#4700 This PR migrates `vulnfeeds/cmd/ids` tool to use the osv-schema Go bindings to replace the deprecated models from osv-scanner. - Replaced `models.Vulnerability` with `osvschema.Vulnerability`. - Switched to `protojson` for JSON serialization, as the bindings are based on protobuf. - Updated YAML processing to use `github.com/goccy/go-yaml`, leveraging a YAML-to-JSON-to-Proto workflow to ensure compatibility. A new test suite is also added to verify ID assignment logic across JSON and YAML formats.
1 parent 15263ea commit bf8f7b4

10 files changed

Lines changed: 331 additions & 29 deletions

File tree

‎vulnfeeds/cmd/ids/main.go‎

Lines changed: 49 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ package main
44
import (
55
"crypto/rand"
66
"encoding/hex"
7-
"encoding/json"
87
"flag"
98
"fmt"
109
"io"
@@ -17,9 +16,10 @@ import (
1716
"strings"
1817
"time"
1918

20-
"github.com/google/osv-scanner/pkg/models"
19+
"github.com/goccy/go-yaml"
2120
"github.com/google/osv/vulnfeeds/utility/logger"
22-
"gopkg.in/yaml.v2"
21+
"github.com/ossf/osv-schema/bindings/go/osvschema"
22+
"google.golang.org/protobuf/encoding/protojson"
2323
)
2424

2525
const (
@@ -112,16 +112,16 @@ func assignID(prefix, path string, format fileFormat, yearCounters map[int]int,
112112
// If the vulnerability has a published date, use the year from that.
113113
// Otherwise, just default to the current year.
114114
year := defaultYear
115-
if !vuln.Published.IsZero() {
116-
year = vuln.Published.Year()
115+
if vuln.GetPublished() != nil {
116+
year = vuln.GetPublished().AsTime().Year()
117117
}
118118

119119
// Allocate a new ID and write the new file.
120120
id := yearCounters[year] + 1
121121
yearCounters[year] = id
122122

123-
vuln.ID = fmt.Sprintf("%s-%d-%d", prefix, year, id)
124-
newPath := filepath.Join(filepath.Dir(path), vuln.ID+formatToExtension[format])
123+
vuln.Id = fmt.Sprintf("%s-%d-%d", prefix, year, id)
124+
newPath := filepath.Join(filepath.Dir(path), vuln.GetId()+formatToExtension[format])
125125

126126
writef, err := os.Create(newPath)
127127
if err != nil {
@@ -193,37 +193,64 @@ func assignIDs(prefix, dir string, format fileFormat) error {
193193
return os.WriteFile(filepath.Join(dir, conflictFile), []byte(hex.EncodeToString(b)), 0600)
194194
}
195195

196-
func readVulnWithFormat(r io.Reader, format fileFormat) (*models.Vulnerability, error) {
197-
var v models.Vulnerability
196+
func readVulnWithFormat(r io.Reader, format fileFormat) (*osvschema.Vulnerability, error) {
197+
data, err := io.ReadAll(r)
198+
if err != nil {
199+
return nil, err
200+
}
201+
202+
var jsonBytes []byte
198203
switch format {
199204
case fileFormatJSON:
200-
dec := json.NewDecoder(r)
201-
if err := dec.Decode(&v); err != nil {
202-
return nil, err
203-
}
205+
jsonBytes = data
204206
case fileFormatYAML:
205-
dec := yaml.NewDecoder(r)
206-
if err := dec.Decode(&v); err != nil {
207+
jsonBytes, err = yaml.YAMLToJSON(data)
208+
if err != nil {
207209
return nil, err
208210
}
209211
default:
210212
return nil, fmt.Errorf("unknown file format: %v", format)
211213
}
212214

215+
var v osvschema.Vulnerability
216+
if err := protojson.Unmarshal(jsonBytes, &v); err != nil {
217+
return nil, err
218+
}
219+
213220
return &v, nil
214221
}
215222

216-
func writeVulnWithFormat(v *models.Vulnerability, w io.Writer, format fileFormat) error {
223+
func writeVulnWithFormat(v *osvschema.Vulnerability, w io.Writer, format fileFormat) error {
224+
marshaler := protojson.MarshalOptions{
225+
Multiline: true,
226+
Indent: " ",
227+
}
228+
229+
jsonBytes, err := marshaler.Marshal(v)
230+
if err != nil {
231+
return err
232+
}
233+
234+
var data []byte
217235
switch format {
218236
case fileFormatJSON:
219-
enc := json.NewEncoder(w)
220-
enc.SetIndent("", " ")
221-
222-
return enc.Encode(v)
237+
data = jsonBytes
223238
case fileFormatYAML:
224-
enc := yaml.NewEncoder(w)
225-
return enc.Encode(v)
239+
data, err = yaml.JSONToYAML(jsonBytes)
240+
if err != nil {
241+
return err
242+
}
226243
default:
227244
return fmt.Errorf("unknown file format: %v", format)
228245
}
246+
247+
// Ensure the output has a trailing newline to match the behavior of
248+
// json.Encoder, which was previously used.
249+
if len(data) > 0 && data[len(data)-1] != '\n' {
250+
data = append(data, '\n')
251+
}
252+
253+
_, err = w.Write(data)
254+
255+
return err
229256
}

‎vulnfeeds/cmd/ids/main_test.go‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
package main
2+
3+
import (
4+
"bytes"
5+
"os"
6+
"path/filepath"
7+
"testing"
8+
)
9+
10+
func TestAssignIDs(t *testing.T) {
11+
tests := []struct {
12+
name string
13+
prefix string
14+
format fileFormat
15+
templateName string
16+
existingName string
17+
expectedID string
18+
}{
19+
{
20+
name: "OSV YAML",
21+
prefix: "OSV",
22+
format: fileFormatYAML,
23+
templateName: "OSV-0000-abc.yaml",
24+
existingName: "OSV-2026-10.yaml",
25+
expectedID: "OSV-2026-11",
26+
},
27+
{
28+
name: "TEST JSON",
29+
prefix: "TEST",
30+
format: fileFormatJSON,
31+
templateName: "TEST-0000-def.json",
32+
existingName: "TEST-2026-20.json",
33+
expectedID: "TEST-2026-21",
34+
},
35+
}
36+
37+
for _, tt := range tests {
38+
t.Run(tt.name, func(t *testing.T) {
39+
tmpDir := t.TempDir()
40+
41+
// 1. Copy existing assigned vulnerability from testdata to set counter.
42+
existingPath := filepath.Join("testdata", tt.existingName)
43+
existingData, err := os.ReadFile(existingPath)
44+
if err != nil {
45+
t.Fatalf("failed to read existing ID: %v", err)
46+
}
47+
err = os.WriteFile(filepath.Join(tmpDir, tt.existingName), existingData, 0600)
48+
if err != nil {
49+
t.Fatalf("failed to copy existing ID: %v", err)
50+
}
51+
52+
// 2. Setup unassigned vulnerability using template.
53+
templatePath := filepath.Join("testdata", tt.templateName)
54+
templateData, err := os.ReadFile(templatePath)
55+
if err != nil {
56+
t.Fatalf("failed to read template %s: %v", templatePath, err)
57+
}
58+
destPath := filepath.Join(tmpDir, tt.templateName)
59+
if err := os.WriteFile(destPath, templateData, 0600); err != nil {
60+
t.Fatalf("failed to setup unassigned vuln: %v", err)
61+
}
62+
63+
// 3. Run assignIDs.
64+
if err := assignIDs(tt.prefix, tmpDir, tt.format); err != nil {
65+
t.Fatalf("assignIDs failed: %v", err)
66+
}
67+
68+
// 4. Verify results.
69+
ext := formatToExtension[tt.format]
70+
expectedFilename := tt.expectedID + ext
71+
gotPath := filepath.Join(tmpDir, expectedFilename)
72+
if _, err := os.Stat(gotPath); os.IsNotExist(err) {
73+
t.Errorf("Expected %s to exist", gotPath)
74+
return
75+
}
76+
77+
if _, err := os.Stat(destPath); !os.IsNotExist(err) {
78+
t.Errorf("Expected old file %s to be removed", tt.templateName)
79+
}
80+
81+
gotData, err := os.ReadFile(gotPath)
82+
if err != nil {
83+
t.Fatalf("failed to read assigned vuln %s: %v", gotPath, err)
84+
}
85+
86+
wantData, err := os.ReadFile(filepath.Join("testdata", expectedFilename))
87+
if err != nil {
88+
t.Fatalf("failed to read expected vuln: %v", err)
89+
}
90+
91+
// Trim space to be robust against minor formatting differences,
92+
// for example trailing newlines.
93+
if !bytes.Equal(bytes.TrimSpace(gotData), bytes.TrimSpace(wantData)) {
94+
t.Errorf("Data fidelity mismatch for %s:\nGot:\n%s\nWant:\n%s", tt.expectedID, string(gotData), string(wantData))
95+
}
96+
97+
// Verify .id-allocator was created.
98+
if _, err := os.Stat(filepath.Join(tmpDir, ".id-allocator")); os.IsNotExist(err) {
99+
t.Errorf("Expected .id-allocator to exist")
100+
}
101+
})
102+
}
103+
}

‎vulnfeeds/cmd/ids/testdata/OSV-0000-abc.yaml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
id:
2+
published: "2026-01-02T00:00:00Z"
3+
modified: "2026-01-02T00:00:00Z"
4+
summary: A vulnerability
5+
details: |
6+
Blah blah blah
7+
Blah
8+
affected:
9+
- package:
10+
name: blah.com/package
11+
ecosystem: Go
12+
ranges:
13+
- type: GIT
14+
repo: https://osv-test/repo/url
15+
events:
16+
- introduced: eefe8ec3f1f90d0e684890e810f3f21e8500a4cd
17+
- fixed: 8d8242f545e9cec3e6d0d2e3f5bde8be1c659735
18+
versions:
19+
- branch-v0.1.1
20+
references:
21+
- type: WEB
22+
url: https://ref.com/ref
23+
database_specific:
24+
specific: 1337
25+
severity:
26+
- type: CVSS_V3
27+
score: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:L/A:L
28+
credits:
29+
- name: Foo bar
30+
contact:
31+
- mailto:foo@bar.com

‎vulnfeeds/cmd/ids/testdata/OSV-2026-10.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
id: OSV-2026-10
2+
modified: '2026-01-01T00:00:00Z'

‎vulnfeeds/cmd/ids/testdata/OSV-2026-11.yaml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
id: OSV-2026-11
2+
published: "2026-01-02T00:00:00Z"
3+
modified: "2026-01-02T00:00:00Z"
4+
summary: A vulnerability
5+
details: |
6+
Blah blah blah
7+
Blah
8+
affected:
9+
- package:
10+
name: blah.com/package
11+
ecosystem: Go
12+
ranges:
13+
- type: GIT
14+
repo: https://osv-test/repo/url
15+
events:
16+
- introduced: eefe8ec3f1f90d0e684890e810f3f21e8500a4cd
17+
- fixed: 8d8242f545e9cec3e6d0d2e3f5bde8be1c659735
18+
versions:
19+
- branch-v0.1.1
20+
references:
21+
- type: WEB
22+
url: https://ref.com/ref
23+
database_specific:
24+
specific: 1337
25+
severity:
26+
- type: CVSS_V3
27+
score: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:L/A:L
28+
credits:
29+
- name: Foo bar
30+
contact:
31+
- mailto:foo@bar.com

‎vulnfeeds/cmd/ids/testdata/TEST-0000-def.json‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
{
2+
"id": "",
3+
"published": "2026-01-02T00:00:00Z",
4+
"modified": "2026-01-02T00:00:00Z",
5+
"summary": "A vulnerability",
6+
"details": "Blah blah blah\nBlah\n",
7+
"affected": [
8+
{
9+
"package": {
10+
"name": "blah.com/package",
11+
"ecosystem": "Go"
12+
},
13+
"ranges": [
14+
{
15+
"type": "GIT",
16+
"repo": "https://osv-test/repo/url",
17+
"events": [
18+
{
19+
"introduced": "eefe8ec3f1f90d0e684890e810f3f21e8500a4cd"
20+
},
21+
{
22+
"fixed": "8d8242f545e9cec3e6d0d2e3f5bde8be1c659735"
23+
}
24+
]
25+
}
26+
],
27+
"versions": [
28+
"branch-v0.1.1"
29+
]
30+
}
31+
],
32+
"references": [
33+
{
34+
"type": "WEB",
35+
"url": "https://ref.com/ref"
36+
}
37+
],
38+
"database_specific": {
39+
"specific": 1337
40+
},
41+
"severity": [
42+
{
43+
"type": "CVSS_V3",
44+
"score": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:L/A:L"
45+
}
46+
],
47+
"credits": [
48+
{
49+
"name": "Foo bar",
50+
"contact": [
51+
"mailto:foo@bar.com"
52+
]
53+
}
54+
]
55+
}

‎vulnfeeds/cmd/ids/testdata/TEST-2026-20.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"id": "TEST-2026-20",
3+
"modified": "2026-01-01T00:00:00Z"
4+
}

0 commit comments

Comments
 (0)