From 31be5f74ea3cfd49e14ff913c764ffa0801c1c46 Mon Sep 17 00:00:00 2001 From: Udan Jayanith Jayakody Date: Sat, 9 Aug 2025 22:28:56 +0530 Subject: [PATCH 1/3] Add selector tokanization and tests for it --- FUTURE-CHANGELOG.md | 1 - querying.go | 68 ++++++++++++++++++++++++++++++++++++++------- querying_test.go | 32 +++++++++++++++++++-- 3 files changed, 88 insertions(+), 13 deletions(-) diff --git a/FUTURE-CHANGELOG.md b/FUTURE-CHANGELOG.md index 0db71dd..fd94fb4 100644 --- a/FUTURE-CHANGELOG.md +++ b/FUTURE-CHANGELOG.md @@ -8,6 +8,5 @@ * GetElementsByClassName * GetElementsByTagName * GetElementsById -* QueryList * QuerySelector * QuerySelectorAll \ No newline at end of file diff --git a/querying.go b/querying.go index aa3f2f0..6cf54e2 100644 --- a/querying.go +++ b/querying.go @@ -39,12 +39,12 @@ func (node *Node) GetElementByClassName(className string) *Node { } // GetElementByID returns the first node that match with the given idName by advancing from the node. -func (node *Node) GetElementByID(idName string) *Node{ +func (node *Node) GetElementByID(idName string) *Node { traverser := NewTraverser(node) var returnNode *Node traverser.Walkthrough(func(node *Node) TraverseCondition { - id, _ := node.GetAttribute("id") - if id == idName{ + id, _ := node.GetAttribute("id") + if id == idName { returnNode = node return StopWalkthrough } @@ -54,7 +54,7 @@ func (node *Node) GetElementByID(idName string) *Node{ } // GetElementsByClassName returns a NodeList containing nodes that have the given className from the node. -func (node *Node) GetElementsByClassName(className string) NodeList{ +func (node *Node) GetElementsByClassName(className string) NodeList { traverser := NewTraverser(node) nodeList := NewNodeList() @@ -62,7 +62,7 @@ func (node *Node) GetElementsByClassName(className string) NodeList{ classList := NewClassList() classList.DecodeFrom(node) - if classList.Contains(className){ + if classList.Contains(className) { nodeList.Append(node) } return ContinueWalkthrough @@ -71,12 +71,12 @@ func (node *Node) GetElementsByClassName(className string) NodeList{ } // GetElementsByTagName returns a NodeList containing nodes that have the given tagName from the node. -func (node *Node) GetElementsByTagName(tagName string) NodeList{ +func (node *Node) GetElementsByTagName(tagName string) NodeList { traverser := NewTraverser(node) nodeList := NewNodeList() traverser.Walkthrough(func(node *Node) TraverseCondition { - if node.GetTagName() == tagName{ + if node.GetTagName() == tagName { nodeList.Append(node) } return ContinueWalkthrough @@ -85,16 +85,64 @@ func (node *Node) GetElementsByTagName(tagName string) NodeList{ } // GetElementsByClassName returns a NodeList containing nodes that have the given idName from the node. -func (node *Node) GetElementsById(idName string) NodeList{ +func (node *Node) GetElementsById(idName string) NodeList { traverser := NewTraverser(node) nodeList := NewNodeList() traverser.Walkthrough(func(node *Node) TraverseCondition { id, _ := node.GetAttribute("id") - if id == idName{ + if id == idName { nodeList.Append(node) } return ContinueWalkthrough }) return nodeList -} \ No newline at end of file +} + +// Selector types +const ( + Id int = iota + Tag + Class +) + +// SelectorToken store data about basic css selectors(ids, classes, tags). +type SelectorToken struct { + Type int + SelectorName string + Selector string +} + +// TokenizeSelector returns a []SelectorToken in selector. +func TokenizeSelector(selector string) []SelectorToken { + slice := make([]SelectorToken, 0, 1) + if strings.TrimSpace(selector) == "" { + return slice + } + + iter := strings.SplitSeq(selector, " ") + for sec := range iter { + token := SelectorToken{} + switch sec{ + case "", " ", ".", "#": + continue + } + + switch string(sec[0]) { + case ".": + token.Type = Class + token.SelectorName = sec[1:] + case "#": + token.Type = Id + token.SelectorName = sec[1:] + default: + token.Type = Tag + token.SelectorName = sec + } + token.Selector = sec + slice = append(slice, token) + } + + return slice +} + diff --git a/querying_test.go b/querying_test.go index f3fed28..3fc94af 100644 --- a/querying_test.go +++ b/querying_test.go @@ -52,7 +52,7 @@ func TestGetElementByClassName(t *testing.T) { node = node.GetElementByClassName("ordered-item") if node == nil { t.Fatal("Node is nil") - }else if node.GetInnerText() != "Apple" { + } else if node.GetInnerText() != "Apple" { t.Fatal("Expected Apple but got ", node.GetInnerText()) } } @@ -66,7 +66,7 @@ func TestGetElementByTagName(t *testing.T) { node = node.GetElementByTagName("h2") if node == nil { t.Fatal("Node is nil") - }else if node.GetInnerText() != "List 1"{ + } else if node.GetInnerText() != "List 1" { t.Fatal("Expected List 1 but got ", node.GetInnerText()) } } @@ -137,3 +137,31 @@ func TestGetElementsById(t *testing.T) { } } } + +func TestSelectorTokenizer(t *testing.T) { + stack := linkedliststack.New() + stack.Push("article .content") + stack.Push("article p h1") + stack.Push("article p") + stack.Push(".title #user") + stack.Push("#user title .title-1") + + for stack.Size() > 0 { + val, _ := stack.Pop() + selector := val.(string) + + tokens := GoHtml.TokenizeSelector(selector) + s := "" + for _, token := range tokens { + if s == "" { + s += token.Selector + }else{ + s += " " + token.Selector + } + } + + if s != selector{ + t.Fatal("Expected ", selector, "but got", s) + } + } +} From ca4d6a9493d22cc100471361fb9508f8424f2dbc Mon Sep 17 00:00:00 2001 From: Udan Jayanith Date: Thu, 14 Aug 2025 16:09:10 +0530 Subject: [PATCH 2/3] New querying functions, traverser walkthrough improvements and node attributes setters and getters improvements --- node-tree.go | 7 +++-- querying.go | 73 +++++++++++++++++++++++++++++++++++++++++------ querying_test.go | 33 +++++++++++++++++++-- traverser.go | 13 +++++---- traverser_test.go | 2 +- 5 files changed, 106 insertions(+), 22 deletions(-) diff --git a/node-tree.go b/node-tree.go index 451f0b6..759a0a7 100644 --- a/node-tree.go +++ b/node-tree.go @@ -68,13 +68,14 @@ func (node *Node) SetTagName(tagName string) { // GetAttribute returns the specified attribute value form the node. If the specified attribute doesn't exists GetAttribute returns a empty string and false. func (node *Node) GetAttribute(attributeName string) (string, bool) { - v, ok := node.attributes[attributeName] + v, ok := node.attributes[strings.TrimSpace(strings.ToLower(attributeName))] return v, ok } // RemoveAttribute remove or delete the specified attribute. func (node *Node) RemoveAttribute(attributeName string) { - delete(node.attributes, attributeName) + delete(node.attributes, strings.TrimSpace(strings.ToLower(attributeName))) + } // IterateAttributes calls callback at every attribute in the node by passing attribute and value of the node. @@ -87,7 +88,7 @@ func (node *Node) IterateAttributes(callback func(attribute, value string)) { // SetAttribute add a attribute to the node. func (node *Node) SetAttribute(attribute, value string) { - node.attributes[strings.TrimSpace(attribute)] = strings.TrimSpace(value) + node.attributes[strings.ToLower(strings.TrimSpace(attribute))] = strings.TrimSpace(value) } // GetText returns text on the node. This does not returns text on it's child nodes. If you also wants child nodes text use GetInnerText method on the node. diff --git a/querying.go b/querying.go index 6cf54e2..88a6409 100644 --- a/querying.go +++ b/querying.go @@ -106,24 +106,24 @@ const ( Class ) -// SelectorToken store data about basic css selectors(ids, classes, tags). -type SelectorToken struct { +// QueryToken store data about basic css selectors(ids, classes, tags). +type QueryToken struct { Type int SelectorName string Selector string } -// TokenizeSelector returns a []SelectorToken in selector. -func TokenizeSelector(selector string) []SelectorToken { - slice := make([]SelectorToken, 0, 1) - if strings.TrimSpace(selector) == "" { +// TokenizeQuery tokenizes the query and returns a list of QueryToken. +func TokenizeQuery(query string) []QueryToken { + slice := make([]QueryToken, 0, 1) + if strings.TrimSpace(query) == "" { return slice } - iter := strings.SplitSeq(selector, " ") + iter := strings.SplitSeq(query, " ") for sec := range iter { - token := SelectorToken{} - switch sec{ + token := QueryToken{} + switch sec { case "", " ", ".", "#": continue } @@ -146,3 +146,58 @@ func TokenizeSelector(selector string) []SelectorToken { return slice } +func matchQueryTokens(node *Node, queryTokens []QueryToken) bool { + if len(queryTokens) == 0 { + return false + } + classList := NewClassList() + classList.DecodeFrom(node) + for _, token := range queryTokens { + switch token.Type { + case Id: + idName, _ := node.GetAttribute("id") + if token.SelectorName != idName { + return false + } + case Tag: + if node.GetTagName() != token.SelectorName { + return false + } + case Class: + if !classList.Contains(token.SelectorName) { + return false + } + } + } + return true +} + +// QuerySelector returns the first node that matches with the give node. +func (node *Node) QuerySelector(query string) *Node { + queryTokens := TokenizeQuery(query) + + traverser := NewTraverser(node) + var res *Node + traverser.Walkthrough(func(node *Node) TraverseCondition { + if matchQueryTokens(node, queryTokens) { + res = node + return StopWalkthrough + } + return ContinueWalkthrough + }) + return res +} + +// QuerySelectorAll returns a NodeList containing nodes that matched with the given query. +func (node *Node) QuerySelectorAll(query string) NodeList{ + nodeList := NewNodeList() + queryTokens := TokenizeQuery(query) + traverser := NewTraverser(node) + + for node := range traverser.Walkthrough{ + if matchQueryTokens(node, queryTokens) { + nodeList.Append(node) + } + } + return nodeList +} \ No newline at end of file diff --git a/querying_test.go b/querying_test.go index 3fc94af..26afac7 100644 --- a/querying_test.go +++ b/querying_test.go @@ -150,18 +150,45 @@ func TestSelectorTokenizer(t *testing.T) { val, _ := stack.Pop() selector := val.(string) - tokens := GoHtml.TokenizeSelector(selector) + tokens := GoHtml.TokenizeQuery(selector) s := "" for _, token := range tokens { if s == "" { s += token.Selector - }else{ + } else { s += " " + token.Selector } } - if s != selector{ + if s != selector { t.Fatal("Expected ", selector, "but got", s) } } } + +func TestQuerySelector(t *testing.T) { + node, err := testFile4NodeTree() + if err != nil { + t.Fatal(err) + return + } + imgEl := node.QuerySelector("img #idElement") + imgSrc, _ := imgEl.GetAttribute("src") + imgAlt, _ := imgEl.GetAttribute("alt") + if imgSrc != "" || imgAlt != "" { + t.Fatal("") + } +} + +func TestQuerySelectorAll(t *testing.T) { + node, err := testFile4NodeTree() + if err != nil { + t.Fatal(err) + return + } + + nodeList := node.QuerySelectorAll("h2") + if nodeList.Len() != 2{ + t.Fatal("") + } +} diff --git a/traverser.go b/traverser.go index d958273..6bec74f 100644 --- a/traverser.go +++ b/traverser.go @@ -42,15 +42,16 @@ func (t *Traverser) Previous() *Node { return t.GetCurrentNode() } -type TraverseCondition bool +type TraverseCondition = bool const ( - StopWalkthrough TraverseCondition = true - ContinueWalkthrough TraverseCondition = false + StopWalkthrough TraverseCondition = false + ContinueWalkthrough TraverseCondition = true ) -// Walkthrough traverse the node tree from the current node to the end of the node tree by visiting every node. If callback returned StopWalkthrough walkthrough function will stop else if it returned ContinueWalkthrough it advanced to the next node. -// Walkthrough calls callback at every node and pass that node. Walkthrough traverse the node tree similar to DFS without visiting visited nodes iteratively. +// Walkthrough traverse the node tree from the current node to the end of the node tree by visiting every node. +// Walkthrough traverse the node tree similar to DFS without visiting visited nodes iteratively. +// Walkthrough can be used as a range over iterator or a function that takes a callback and pass every node one by one. func (t *Traverser) Walkthrough(callback func(node *Node) TraverseCondition) { stack := linkedliststack.New() if t.GetCurrentNode() == nil { @@ -60,7 +61,7 @@ func (t *Traverser) Walkthrough(callback func(node *Node) TraverseCondition) { for stack.Size() > 0 { currentNode, _ := stack.Pop() - if callback(currentNode.(*Node)) == StopWalkthrough { + if !callback(currentNode.(*Node)) { return } diff --git a/traverser_test.go b/traverser_test.go index aaecbd2..95ed27c 100644 --- a/traverser_test.go +++ b/traverser_test.go @@ -19,7 +19,7 @@ func TestWalkthrough(t *testing.T) { traverser := GoHtml.NewTraverser(body) resList := make([]*GoHtml.Node, 0) - traverser.Walkthrough(func(node *GoHtml.Node) GoHtml.TraverseCondition { + traverser.Walkthrough(func(node *GoHtml.Node) bool { resList = append(resList, node) return GoHtml.ContinueWalkthrough }) From 65665d6de1ab93b4783923455236c6b4f9ce5670 Mon Sep 17 00:00:00 2001 From: Udan Jayanith Date: Thu, 14 Aug 2025 17:07:23 +0530 Subject: [PATCH 3/3] Updated go version --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 185d004..c532094 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/udan-jayanith/GoHTML -go 1.24.1 +go 1.25.0 require github.com/emirpasic/gods v1.18.1