fix(conda): add support pip deps for environment.yml files (#6675)

Author: DmitriyLewen

pkg/dependency/parser/conda/environment/parse.go

@@ -15,7 +15,11 @@ import (
 )
 
 type environment struct {
-	Dependencies []Dependency `yaml:"dependencies"`
+	Entries []Entry `yaml:"dependencies"`
+}
+
+type Entry struct {
+	Dependencies []Dependency
 }
 
 type Dependency struct {
@@ -42,13 +46,15 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependenc
 	}
 
 	var pkgs ftypes.Packages
-	for _, dep := range env.Dependencies {
-		pkg := p.toPackage(dep)
-		// Skip empty pkgs
-		if pkg.Name == "" {
-			continue
+	for _, entry := range env.Entries {
+		for _, dep := range entry.Dependencies {
+			pkg := p.toPackage(dep)
+			// Skip empty pkgs
+			if pkg.Name == "" {
+				continue
+			}
+			pkgs = append(pkgs, pkg)
 		}
-		pkgs = append(pkgs, pkg)
 	}
 
 	sort.Sort(pkgs)
@@ -96,8 +102,40 @@ func (*Parser) parseDependency(line string) (string, string) {
 	return name, parts[1]
 }
 
-func (d *Dependency) UnmarshalYAML(node *yaml.Node) error {
-	d.Value = node.Value
-	d.Line = node.Line
+func (e *Entry) UnmarshalYAML(node *yaml.Node) error {
+	var dependencies []Dependency
+	// cf. https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/resolve.go#L70-L81
+	switch node.Tag {
+	case "!!str":
+		dependencies = append(dependencies, Dependency{
+			Value: node.Value,
+			Line:  node.Line,
+		})
+	case "!!map":
+		if node.Content != nil {
+			// Map key is package manager (e.g. pip). So we need to store only map values (dependencies).
+			// e.g. dependencies:
+			//  	  - pip:
+			//     	    - pandas==2.1.4
+			if node.Content[1].Tag != "!!seq" { // Conda supports only map[string][]string format.
+				return xerrors.Errorf("unsupported dependency type %q on line %d", node.Content[1].Tag, node.Content[1].Line)
+			}
+
+			for _, depContent := range node.Content[1].Content {
+				if depContent.Tag != "!!str" {
+					return xerrors.Errorf("unsupported dependency type %q on line %d", depContent.Tag, depContent.Line)
+				}
+
+				dependencies = append(dependencies, Dependency{
+					Value: depContent.Value,
+					Line:  depContent.Line,
+				})
+			}
+		}
+	default:
+		return xerrors.Errorf("unsupported dependency type %q on line %d", node.Tag, node.Line)
+	}
+
+	e.Dependencies = dependencies
 	return nil
 }

pkg/dependency/parser/conda/environment/parse_test.go

@@ -31,6 +31,16 @@ func TestParse(t *testing.T) {
 						},
 					},
 				},
+				{
+					Name:    "asgiref",
+					Version: "3.8.1",
+					Locations: ftypes.Locations{
+						{
+							StartLine: 21,
+							EndLine:   21,
+						},
+					},
+				},
 				{
 					Name:    "blas",
 					Version: "1.0",
@@ -61,6 +71,16 @@ func TestParse(t *testing.T) {
 						},
 					},
 				},
+				{
+					Name:    "django",
+					Version: "5.0.6",
+					Locations: ftypes.Locations{
+						{
+							StartLine: 22,
+							EndLine:   22,
+						},
+					},
+				},
 				{
 					Name: "ld_impl_linux-aarch64",
 					Locations: ftypes.Locations{
@@ -167,9 +187,24 @@ func TestParse(t *testing.T) {
 			},
 		},
 		{
-			name:    "invalid_json",
+			name:    "invalid yaml file",
 			input:   "testdata/invalid.yaml",
-			wantErr: "unable to decode conda environment.yml file",
+			wantErr: "cannot unmarshal !!str `invalid` into environment.environment",
+		},
+		{
+			name:    "`dependency` field uses unsupported type",
+			input:   "testdata/wrong-deps-type.yaml",
+			wantErr: `unsupported dependency type "!!int" on line 5`,
+		},
+		{
+			name:    "nested field uses unsupported type",
+			input:   "testdata/wrong-nested-type.yaml",
+			wantErr: `unsupported dependency type "!!str" on line 5`,
+		},
+		{
+			name:    "nested dependency uses unsupported type",
+			input:   "testdata/wrong-nested-dep-type.yaml",
+			wantErr: `unsupported dependency type "!!map" on line 6`,
 		},
 	}
 	for _, tt := range tests {

pkg/dependency/parser/conda/environment/testdata/happy.yaml

@@ -17,5 +17,8 @@ dependencies:
   - liblapack=3.9.*=22_linuxaarch64_openblas
   - libnsl=2.0.1=h31becfc_0
   - bzip2=1.0.8=h998d150_5
+  - pip:
+      - asgiref==3.8.1
+      - django==5.0.6
 
 prefix: /opt/conda/envs/test-env

pkg/dependency/parser/conda/environment/testdata/wrong-deps-type.yaml

@@ -0,0 +1,7 @@
+name: test-env
+channels:
+  - defaults
+dependencies:
+  - 1
+
+prefix: /opt/conda/envs/test-env

pkg/dependency/parser/conda/environment/testdata/wrong-nested-dep-type.yaml

@@ -0,0 +1,9 @@
+name: test-env
+channels:
+  - defaults
+dependencies:
+  - pip:
+    - wrongType:
+      - asgiref==3.8.1
+
+prefix: /opt/conda/envs/test-env

pkg/dependency/parser/conda/environment/testdata/wrong-nested-type.yaml

@@ -0,0 +1,7 @@
+name: test-env
+channels:
+  - defaults
+dependencies:
+  - pip: asgiref==3.8.1
+
+prefix: /opt/conda/envs/test-env