diff --git a/server/build.gradle b/server/build.gradle index 01a2439..bb745c5 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -34,18 +34,35 @@ dependencies { implementation 'com.bucket4j:bucket4j_jdk17-redis-common:8.14.0' implementation 'com.bucket4j:bucket4j_jdk17-lettuce:8.14.0' implementation 'io.jsonwebtoken:jjwt-api:0.13.0' + implementation 'org.springframework.boot:spring-boot-starter-data-jdbc' runtimeOnly 'io.jsonwebtoken:jjwt-impl:0.13.0' runtimeOnly 'io.jsonwebtoken:jjwt-jackson:0.13.0' implementation 'org.springframework.boot:spring-boot-starter-data-redis' implementation 'org.flywaydb:flyway-database-postgresql' implementation 'org.springframework.ai:spring-ai-starter-model-ollama' - implementation 'org.springframework.ai:spring-ai-starter-vector-store-pgvector' - implementation 'org.springframework.ai:spring-ai-vector-store-advisor' compileOnly 'org.projectlombok:lombok' implementation 'org.eclipse.jgit:org.eclipse.jgit:7.6.0.202603022253-r' developmentOnly 'org.springframework.boot:spring-boot-devtools' developmentOnly 'org.springframework.boot:spring-boot-docker-compose' + implementation 'com.pgvector:pgvector:0.1.6' runtimeOnly 'org.postgresql:postgresql' + implementation 'io.github.bonede:tree-sitter:0.26.3' + runtimeOnly 'io.github.bonede:tree-sitter-java:0.23.5' + runtimeOnly 'io.github.bonede:tree-sitter-python:0.25.0' + runtimeOnly 'io.github.bonede:tree-sitter-kotlin:0.3.8.1' + runtimeOnly 'io.github.bonede:tree-sitter-javascript:0.25.0' + runtimeOnly 'io.github.bonede:tree-sitter-typescript:0.23.2' + runtimeOnly 'io.github.bonede:tree-sitter-tsx:0.23.2' + runtimeOnly 'io.github.bonede:tree-sitter-go:0.23.3' + runtimeOnly 'io.github.bonede:tree-sitter-rust:0.23.1' + runtimeOnly 'io.github.bonede:tree-sitter-c:0.24.1' + runtimeOnly 'io.github.bonede:tree-sitter-cpp:0.23.4' + runtimeOnly 'io.github.bonede:tree-sitter-c-sharp:0.23.1' + runtimeOnly 'io.github.bonede:tree-sitter-php:0.23.11' + runtimeOnly 'io.github.bonede:tree-sitter-ruby:0.23.1' + runtimeOnly 'io.github.bonede:tree-sitter-swift:0.5.0' + runtimeOnly 'io.github.bonede:tree-sitter-html:0.23.2' + runtimeOnly 'io.github.bonede:tree-sitter-css:0.25.0' developmentOnly 'org.springframework.ai:spring-ai-spring-boot-docker-compose' annotationProcessor 'org.projectlombok:lombok' testImplementation 'org.springframework.boot:spring-boot-starter-data-jpa-test' diff --git a/server/src/main/java/com/meet/server/ServerApplication.java b/server/src/main/java/com/meet/server/ServerApplication.java index 25361e7..b4ffb73 100644 --- a/server/src/main/java/com/meet/server/ServerApplication.java +++ b/server/src/main/java/com/meet/server/ServerApplication.java @@ -7,10 +7,12 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.data.jpa.repository.config.EnableJpaAuditing; import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.EnableAsync; @SpringBootApplication @EnableJpaAuditing @EnableScheduling +@EnableAsync public class ServerApplication { private static final Logger log = LogManager.getLogger(ServerApplication.class); diff --git a/server/src/main/java/com/meet/server/common/config/AppConfig.java b/server/src/main/java/com/meet/server/common/config/AppConfig.java index fad9131..ff606e8 100644 --- a/server/src/main/java/com/meet/server/common/config/AppConfig.java +++ b/server/src/main/java/com/meet/server/common/config/AppConfig.java @@ -1,10 +1,11 @@ package com.meet.server.common.config; import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Bean; import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.security.crypto.password.PasswordEncoder; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; @Configuration public class AppConfig { @@ -26,4 +27,15 @@ public void setCookieSecure(String env) { public PasswordEncoder passwordEncoder() { return new BCryptPasswordEncoder(12); } + + @Bean(name = "codebaseTaskExecutor") + public ThreadPoolTaskExecutor codebaseTaskExecutor() { + var executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(2); + executor.setMaxPoolSize(4); + executor.setQueueCapacity(100); + executor.setThreadNamePrefix("codebase-"); + executor.initialize(); + return executor; + } } diff --git a/server/src/main/java/com/meet/server/common/exception/CodebaseException.java b/server/src/main/java/com/meet/server/common/exception/CodebaseException.java new file mode 100644 index 0000000..b321c79 --- /dev/null +++ b/server/src/main/java/com/meet/server/common/exception/CodebaseException.java @@ -0,0 +1,23 @@ +package com.meet.server.common.exception; + +import lombok.Getter; +import org.springframework.http.HttpStatus; + +@Getter +public class CodebaseException extends RuntimeException { + + private final String errorCode; + private final HttpStatus status; + + public CodebaseException(String errorCode, String message, HttpStatus status) { + super(message); + this.errorCode = errorCode; + this.status = status; + } + + public CodebaseException(String errorCode, String message, HttpStatus status, Throwable cause) { + super(message, cause); + this.errorCode = errorCode; + this.status = status; + } +} diff --git a/server/src/main/java/com/meet/server/common/exception/GlobalExceptionHandler.java b/server/src/main/java/com/meet/server/common/exception/GlobalExceptionHandler.java index fe3bbcf..9889571 100644 --- a/server/src/main/java/com/meet/server/common/exception/GlobalExceptionHandler.java +++ b/server/src/main/java/com/meet/server/common/exception/GlobalExceptionHandler.java @@ -61,6 +61,12 @@ public ResponseEntity> handleAuthException(AuthException excep return response(exception.getStatus(), exception.getMessage()); } + @ExceptionHandler(CodebaseException.class) + public ResponseEntity> handleCodebaseException(CodebaseException exception) { + log.warn("Codebase operation failed [{}]: {}", exception.getErrorCode(), exception.getMessage()); + return response(exception.getStatus(), exception.getMessage()); + } + @ExceptionHandler(InvalidTokenException.class) public ResponseEntity> handleInvalidToken(InvalidTokenException exception) { return response(HttpStatus.UNAUTHORIZED, exception.getMessage()); diff --git a/server/src/main/java/com/meet/server/feature/codebase/CodebaseController.java b/server/src/main/java/com/meet/server/feature/codebase/CodebaseController.java new file mode 100644 index 0000000..3cde576 --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/codebase/CodebaseController.java @@ -0,0 +1,42 @@ +package com.meet.server.feature.codebase; + +import com.meet.server.common.api.ApiResponse; +import com.meet.server.feature.codebase.dto.CodebaseImportRequest; +import com.meet.server.feature.codebase.dto.CodebaseImportResponse; +import jakarta.validation.Valid; +import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.security.core.Authentication; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Optional; +import java.util.UUID; + +@RestController +@RequestMapping("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/api/codebases") +@RequiredArgsConstructor +public class CodebaseController { + + private final CodebaseService codebaseService; + + @PostMapping + public ResponseEntity> importCodebase( + Authentication authentication, + @Valid @RequestBody CodebaseImportRequest request + ) { + var response = codebaseService.startClone( + UUID.fromString(authentication.getName()), + request); + + return ResponseEntity + .status(HttpStatus.ACCEPTED) + .body(new ApiResponse<>( + true, + "Codebase import queued", + Optional.of(response))); + } +} diff --git a/server/src/main/java/com/meet/server/feature/codebase/CodebaseRepository.java b/server/src/main/java/com/meet/server/feature/codebase/CodebaseRepository.java new file mode 100644 index 0000000..ebc7842 --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/codebase/CodebaseRepository.java @@ -0,0 +1,20 @@ +package com.meet.server.feature.codebase; + +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.repository.query.Param; +import org.springframework.data.jpa.repository.Query; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +import java.util.UUID; + +@Repository +public interface CodebaseRepository extends JpaRepository { + + @Modifying + @Transactional + @Query("update Codebase c set c.lastCommitSha = :lastCommitSha where c.id = :codebaseId") + int updateLastCommitSha(@Param("codebaseId") UUID codebaseId, + @Param("lastCommitSha") String lastCommitSha); +} diff --git a/server/src/main/java/com/meet/server/feature/codebase/CodebaseService.java b/server/src/main/java/com/meet/server/feature/codebase/CodebaseService.java new file mode 100644 index 0000000..ade00d8 --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/codebase/CodebaseService.java @@ -0,0 +1,170 @@ +package com.meet.server.feature.codebase; + +import com.meet.server.common.exception.CodebaseException; +import com.meet.server.feature.codebase.dto.CodebaseImportRequest; +import com.meet.server.feature.codebase.dto.CodebaseImportResponse; +import com.meet.server.feature.repositoryfile.RepositoryFileProcessor; +import com.meet.server.feature.user.UserService; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.Inet4Address; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Locale; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; + +@Service +public class CodebaseService { + + private final CodebaseRepository codebaseRepository; + private final UserService userService; + private final GitService gitService; + private final RepositoryFileProcessor fileProcessor; + private final CodebaseStatusService statusService; + private final Executor codebaseTaskExecutor; + + public CodebaseService( + CodebaseRepository codebaseRepository, + UserService userService, + GitService gitService, + RepositoryFileProcessor fileProcessor, + CodebaseStatusService statusService, + @Qualifier("codebaseTaskExecutor") Executor codebaseTaskExecutor + ) { + this.codebaseRepository = codebaseRepository; + this.userService = userService; + this.gitService = gitService; + this.fileProcessor = fileProcessor; + this.statusService = statusService; + this.codebaseTaskExecutor = codebaseTaskExecutor; + } + + @Transactional + public CodebaseImportResponse startClone(UUID userId, CodebaseImportRequest request) { + validateCloneUrl(request.cloneUrl()); + var codebase = codebaseRepository.save(Codebase.builder() + .user(userService.getById(userId)) + .name(request.name()) + .cloneUrl(request.cloneUrl()) + .branch(request.branch() == null || request.branch().isBlank() ? "main" : request.branch()) + .status(CodebaseStatus.QUEUED) + .build()); + + var codebaseId = codebase.getId(); + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCommit() { + processAsync(codebaseId); + } + }); + + return new CodebaseImportResponse(codebaseId, CodebaseStatus.QUEUED, 0); + } + + private void validateCloneUrl(String cloneUrl) { + if (cloneUrl == null || cloneUrl.isBlank()) { + throw invalidCloneUrl(null); + } + final URI uri; + try { + uri = new URI(cloneUrl); + } catch (URISyntaxException exception) { + throw invalidCloneUrl(exception); + } + + if (!"https".equalsIgnoreCase(uri.getScheme()) + || uri.getHost() == null + || uri.getHost().isBlank() + || uri.getUserInfo() != null) { + throw invalidCloneUrl(null); + } + + try { + for (InetAddress address : InetAddress.getAllByName(uri.getHost())) { + if (isInternalAddress(address)) { + throw invalidCloneUrl(null); + } + } + } catch (IOException exception) { + throw invalidCloneUrl(exception); + } + } + + private boolean isInternalAddress(InetAddress address) { + String hostAddress = address.getHostAddress().toLowerCase(Locale.ROOT); + if (address.isAnyLocalAddress() + || address.isLoopbackAddress() + || address.isLinkLocalAddress() + || address.isSiteLocalAddress() + || address.isMulticastAddress() + || hostAddress.startsWith("fc") + || hostAddress.startsWith("fd")) { + return true; + } + if (address instanceof Inet4Address) { + byte[] bytes = address.getAddress(); + int first = bytes[0] & 0xff; + int second = bytes[1] & 0xff; + return first == 100 && second >= 64 && second <= 127 + || first == 192 && second == 0 + || first == 198 && (second == 18 || second == 19) + || first >= 240; + } + return false; + } + + private CodebaseException invalidCloneUrl(Throwable cause) { + return cause == null + ? new CodebaseException("INVALID_CLONE_URL", "Clone URL must be a public HTTPS URL", HttpStatus.BAD_REQUEST) + : new CodebaseException("INVALID_CLONE_URL", "Clone URL must be a public HTTPS URL", HttpStatus.BAD_REQUEST, cause); + } + + public CompletableFuture processAsync(UUID codebaseId) { + return CompletableFuture.supplyAsync(() -> process(codebaseId), codebaseTaskExecutor); + } + + private CodebaseImportResponse process(UUID codebaseId) { + var codebase = codebaseRepository.findById(codebaseId) + .orElseThrow(() -> new CodebaseException( + "CODEBASE_NOT_FOUND", + "Codebase not found", + HttpStatus.NOT_FOUND)); + try { + statusService.update(codebaseId, CodebaseStatus.PROCESSING); + + Path repositoryPath = Files.createTempDirectory("codebase-" + codebaseId); + try { + gitService.cloneRepository(codebase.getCloneUrl(), codebase.getBranch(), repositoryPath); + var commitSha = gitService.currentCommitSha(repositoryPath); + codebaseRepository.updateLastCommitSha(codebaseId, commitSha); + var files = gitService.listFiles(repositoryPath); + var fileCount = fileProcessor.process(codebase, repositoryPath, files, commitSha); + statusService.update(codebaseId, CodebaseStatus.INDEXED); + return new CodebaseImportResponse(codebaseId, CodebaseStatus.INDEXED, fileCount); + } finally { + gitService.deleteRepository(repositoryPath); + } + } catch (IOException exception) { + statusService.update(codebaseId, CodebaseStatus.FAILED); + throw new CodebaseException( + "CODEBASE_WORKSPACE_FAILED", + "Unable to process codebase workspace", + HttpStatus.INTERNAL_SERVER_ERROR, + exception); + } catch (RuntimeException exception) { + statusService.update(codebaseId, CodebaseStatus.FAILED); + throw exception; + } + } +} diff --git a/server/src/main/java/com/meet/server/feature/codebase/CodebaseStatusService.java b/server/src/main/java/com/meet/server/feature/codebase/CodebaseStatusService.java new file mode 100644 index 0000000..3f24f87 --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/codebase/CodebaseStatusService.java @@ -0,0 +1,33 @@ +package com.meet.server.feature.codebase; + +import com.meet.server.common.exception.CodebaseException; +import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; + +import java.time.Instant; +import java.util.UUID; + +@Service +@RequiredArgsConstructor +public class CodebaseStatusService { + + private final CodebaseRepository codebaseRepository; + + @Transactional(propagation = Propagation.REQUIRES_NEW) + public void update(UUID codebaseId, CodebaseStatus status) { + var codebase = codebaseRepository.findById(codebaseId) + .orElseThrow(() -> new CodebaseException( + "CODEBASE_NOT_FOUND", + "Codebase not found", + HttpStatus.NOT_FOUND)); + + codebase.setStatus(status); + if (status == CodebaseStatus.INDEXED) { + codebase.setIndexedAt(Instant.now()); + } + codebaseRepository.save(codebase); + } +} diff --git a/server/src/main/java/com/meet/server/feature/codebase/GitService.java b/server/src/main/java/com/meet/server/feature/codebase/GitService.java index ffd5739..7d9c6d7 100644 --- a/server/src/main/java/com/meet/server/feature/codebase/GitService.java +++ b/server/src/main/java/com/meet/server/feature/codebase/GitService.java @@ -1,29 +1,241 @@ package com.meet.server.feature.codebase; +import com.meet.server.common.exception.CodebaseException; +import com.meet.server.feature.repositoryfile.RepositoryFileDescriptor; +import com.meet.server.feature.indexing.language.Language; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.ignore.IgnoreNode; +import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; -import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.*; @Service public class GitService { - public void clone(String url, String branch, String path) { - try { - var git = Git.cloneRepository() + private static final int CLONE_TIMEOUT_SECONDS = 60; + + private static final Set IGNORED_DIRECTORIES = Set.of( + ".git", ".github", ".gitlab", ".circleci", ".idea", ".vscode", + "node_modules", "bower_components", "vendor", "target", "build", "out", + "dist", "coverage", ".gradle", ".next", ".nuxt", ".angular", "__pycache__", + ".pytest_cache", ".mypy_cache", ".tox", ".venv", "venv", "env", "bin", "obj" + ); + + private static final Set IGNORED_FILE_NAMES = Set.of( + ".gitignore", ".gitattributes", ".gitmodules", ".dockerignore", + ".editorconfig", ".env", ".env.local", ".env.development", ".env.production", + "package-lock.json", "yarn.lock", "pnpm-lock.yaml", "bun.lockb", + "composer.lock", "gemfile.lock", "poetry.lock", "pipfile.lock", "cargo.lock", + "gradle.lockfile", "go.sum" + ); + + private static final Set IGNORED_EXTENSIONS = Set.of( + "7z", "avi", "bmp", "class", "dll", "dmg", "exe", "flac", "gif", "ico", + "jar", "jpeg", "jpg", "m4v", "mkv", "mov", "mp3", "mp4", "mpeg", "mpg", "png", "webm", "svg", + "tar", "ttf", "wav", "webp", "woff", "woff2", "zip", "map", "min.js", "min.css" + ); + + public void cloneRepository(String url, String branch, Path path) { + try (var git = Git.cloneRepository() .setURI(url) - .setBranch(branch) - .setDirectory(new File(path)) - .call(); - git.close(); + .setBranch(branch == null || branch.isBlank() ? "main" : branch) + .setDirectory(path.toFile()) + .setTimeout(CLONE_TIMEOUT_SECONDS) + .call()) { + // The cloned repository is closed when this operation completes. } catch (GitAPIException e) { - throw new RuntimeException(e); + throw new CodebaseException( + "CODEBASE_CLONE_FAILED", + "Unable to clone repository", + HttpStatus.BAD_GATEWAY, + e); + } + } + + public List listFiles(Path repositoryPath) { + var ignoreNodes = loadIgnoreNodes(repositoryPath); + + try (var paths = Files.walk(repositoryPath)) { + return paths + .filter(Files::isRegularFile) + .filter(path -> shouldIndex(repositoryPath, path, ignoreNodes)) + .map(path -> descriptor(repositoryPath, path)) + .toList(); + } catch (IOException e) { + throw new CodebaseException( + "CODEBASE_FILE_LIST_FAILED", + "Unable to list repository files", + HttpStatus.INTERNAL_SERVER_ERROR, + e); + } + } + + public String currentCommitSha(Path repositoryPath) { + try (var git = Git.open(repositoryPath.toFile())) { + ObjectId head = git.getRepository().resolve("HEAD"); + if (head == null) { + throw new IOException("Repository HEAD is not available"); + } + return head.name(); + } catch (IOException e) { + throw new CodebaseException( + "CODEBASE_COMMIT_SHA_FAILED", + "Unable to resolve repository commit", + HttpStatus.INTERNAL_SERVER_ERROR, + e); } } - public void delete(String path) { - File file = new File(path); - file.delete(); + private Map loadIgnoreNodes(Path repositoryPath) { + try (var paths = Files.walk(repositoryPath)) { + return paths.filter(Files::isDirectory) + .map(Path::normalize) + .filter(path -> Files.isRegularFile(path.resolve(".gitignore"))) + .collect(LinkedHashMap::new, (nodes, path) -> nodes.put(path, loadIgnoreNode(path)), Map::putAll); + } catch (IOException e) { + throw new CodebaseException( + "CODEBASE_IGNORE_FILE_FAILED", + "Unable to read repository ignore rules", + HttpStatus.INTERNAL_SERVER_ERROR, + e); + } + } + + private IgnoreNode loadIgnoreNode(Path directory) { + var ignoreNode = new IgnoreNode(); + try (InputStream input = Files.newInputStream(directory.resolve(".gitignore"))) { + ignoreNode.parse(input); + return ignoreNode; + } catch (IOException e) { + throw new CodebaseException( + "CODEBASE_IGNORE_FILE_FAILED", + "Unable to read repository ignore rules", + HttpStatus.INTERNAL_SERVER_ERROR, + e); + } + } + + private boolean shouldIndex(Path repositoryPath, Path file, Map ignoreNodes) { + var relativePath = repositoryPath.relativize(file).toString().replace('\\', '/'); + var pathParts = relativePath.split("/"); + var fileName = pathParts[pathParts.length - 1].toLowerCase(Locale.ROOT); + + for (int index = 0; index < pathParts.length - 1; index++) { + if (IGNORED_DIRECTORIES.contains(pathParts[index].toLowerCase(Locale.ROOT))) { + return false; + } + } + + if (IGNORED_FILE_NAMES.contains(fileName) || isIgnoredExtension(fileName)) { + return false; + } + + for (Path directory = repositoryPath; directory != null && directory.startsWith(repositoryPath); + directory = directory.equals(repositoryPath) ? null : directory.getParent()) { + var ignoreNode = ignoreNodes.get(directory.normalize()); + if (ignoreNode != null) { + var ignoredPath = directory.relativize(file).toString().replace('\\', '/'); + var pathPartsFromDirectory = ignoredPath.split("/"); + StringBuilder path = new StringBuilder(); + for (int index = 0; index < pathPartsFromDirectory.length - 1; index++) { + if (path.length() > 0) { + path.append('/'); + } + path.append(pathPartsFromDirectory[index]); + if (Boolean.TRUE.equals(ignoreNode.checkIgnored(path.toString(), true))) { + return false; + } + } + if (Boolean.TRUE.equals(ignoreNode.checkIgnored(ignoredPath, false))) { + return false; + } + } + } + return true; + } + + private boolean isIgnoredExtension(String fileName) { + var dot = fileName.lastIndexOf('.'); + if (dot < 0) { + return false; + } + + var extension = fileName.substring(dot + 1); + return IGNORED_EXTENSIONS.contains(extension) + || IGNORED_EXTENSIONS.stream().anyMatch(fileName::endsWith); + } + + public void deleteRepository(Path repositoryPath) { + if (repositoryPath == null || !Files.exists(repositoryPath)) { + return; + } + + try (var paths = Files.walk(repositoryPath)) { + paths.sorted(Comparator.reverseOrder()).forEach(this::deletePath); + } catch (IOException e) { + throw new CodebaseException( + "CODEBASE_CLEANUP_FAILED", + "Unable to delete cloned repository", + HttpStatus.INTERNAL_SERVER_ERROR, + e); + } + } + + private RepositoryFileDescriptor descriptor(Path repositoryPath, Path file) { + try { + var relativePath = repositoryPath.relativize(file).toString().replace('\\', '/'); + return new RepositoryFileDescriptor( + relativePath, + Language.extensionOf(relativePath), + Files.size(file), + sha256(file)); + } catch (IOException e) { + throw new CodebaseException( + "CODEBASE_FILE_INSPECTION_FAILED", + "Unable to inspect repository file", + HttpStatus.INTERNAL_SERVER_ERROR, + e); + } + } + + private String sha256(Path file) { + try { + var digest = MessageDigest.getInstance("SHA-256"); + try (var input = Files.newInputStream(file)) { + var buffer = new byte[8192]; + int read; + while ((read = input.read(buffer)) != -1) { + digest.update(buffer, 0, read); + } + } + return HexFormat.of().formatHex(digest.digest()); + } catch (IOException | NoSuchAlgorithmException e) { + throw new CodebaseException( + "CODEBASE_CHECKSUM_FAILED", + "Unable to checksum repository file", + HttpStatus.INTERNAL_SERVER_ERROR, + e); + } + } + + private void deletePath(Path path) { + try { + Files.deleteIfExists(path); + } catch (IOException e) { + throw new CodebaseException( + "CODEBASE_CLEANUP_FAILED", + "Unable to delete cloned repository path", + HttpStatus.INTERNAL_SERVER_ERROR, + e); + } } } diff --git a/server/src/main/java/com/meet/server/feature/codebase/dto/CodebaseImportRequest.java b/server/src/main/java/com/meet/server/feature/codebase/dto/CodebaseImportRequest.java new file mode 100644 index 0000000..c04a9a1 --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/codebase/dto/CodebaseImportRequest.java @@ -0,0 +1,13 @@ +package com.meet.server.feature.codebase.dto; + +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.Pattern; + +public record CodebaseImportRequest( + @NotBlank String name, + @NotBlank + @Pattern(regexp = "(?i)^https://.+$", message = "cloneUrl must use HTTPS") + String cloneUrl, + String branch +) { +} diff --git a/server/src/main/java/com/meet/server/feature/codebase/dto/CodebaseImportResponse.java b/server/src/main/java/com/meet/server/feature/codebase/dto/CodebaseImportResponse.java new file mode 100644 index 0000000..9532f2d --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/codebase/dto/CodebaseImportResponse.java @@ -0,0 +1,12 @@ +package com.meet.server.feature.codebase.dto; + +import com.meet.server.feature.codebase.CodebaseStatus; + +import java.util.UUID; + +public record CodebaseImportResponse( + UUID codebaseId, + CodebaseStatus status, + int fileCount +) { +} diff --git a/server/src/main/java/com/meet/server/feature/codechunk/CodeChunk.java b/server/src/main/java/com/meet/server/feature/codechunk/CodeChunk.java index e6209e8..8973035 100644 --- a/server/src/main/java/com/meet/server/feature/codechunk/CodeChunk.java +++ b/server/src/main/java/com/meet/server/feature/codechunk/CodeChunk.java @@ -44,7 +44,7 @@ public class CodeChunk extends BaseAuditEntity { @Column(name = "chunk_index", nullable = false) private Integer chunkIndex; - + @Column(nullable = false, columnDefinition = "TEXT") private String content; @@ -62,5 +62,4 @@ public class CodeChunk extends BaseAuditEntity { private Integer endLine; private String commitSha; - } diff --git a/server/src/main/java/com/meet/server/feature/codechunk/CodeChunkRepository.java b/server/src/main/java/com/meet/server/feature/codechunk/CodeChunkRepository.java index b426cbe..2e71cd4 100644 --- a/server/src/main/java/com/meet/server/feature/codechunk/CodeChunkRepository.java +++ b/server/src/main/java/com/meet/server/feature/codechunk/CodeChunkRepository.java @@ -1,10 +1,25 @@ package com.meet.server.feature.codechunk; -import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.stereotype.Repository; +import com.meet.server.feature.embedding.SimilaritySearchRequest; +import java.util.Collection; +import java.util.List; +import java.util.Optional; import java.util.UUID; -@Repository -public interface CodeChunkRepository extends JpaRepository { +public interface CodeChunkRepository { + + void saveAll(Collection chunks); + + void updateEmbedding(UUID chunkId, float[] embedding); + + void deleteByFileId(UUID fileId); + + void deleteByCodebaseId(UUID codebaseId); + + Optional findById(UUID chunkId); + + List similaritySearch(SimilaritySearchRequest request); + + long countByCodebaseId(UUID codebaseId); } diff --git a/server/src/main/java/com/meet/server/feature/codechunk/CodeChunkRepositoryImpl.java b/server/src/main/java/com/meet/server/feature/codechunk/CodeChunkRepositoryImpl.java new file mode 100644 index 0000000..3bc5586 --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/codechunk/CodeChunkRepositoryImpl.java @@ -0,0 +1,284 @@ +package com.meet.server.feature.codechunk; + +import com.meet.server.feature.codebase.Codebase; +import com.meet.server.feature.embedding.SimilaritySearchRequest; +import com.meet.server.feature.repositoryfile.RepositoryFile; +import com.pgvector.PGvector; +import lombok.RequiredArgsConstructor; +import org.postgresql.util.PGobject; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.simple.JdbcClient; +import org.springframework.stereotype.Repository; + +import java.sql.*; +import java.time.Instant; +import java.util.HashMap; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; + +@Repository +@RequiredArgsConstructor +public class CodeChunkRepositoryImpl implements CodeChunkRepository { + + private static final int BATCH_SIZE = 500; + private static final String CHUNK_COLUMNS = """ + c.id, c.created_at, c.updated_at, c.file_id, c.codebase_id, + c.chunk_index, c.content, c.embedding, c.language, c.path, + c.start_line, c.end_line, c.commit_sha + """; + + private static final String INSERT_SQL = """ + INSERT INTO code_chunks ( + id, created_at, updated_at, file_id, codebase_id, chunk_index, + content, embedding, language, path, start_line, end_line, commit_sha + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (file_id, chunk_index) DO UPDATE SET + updated_at = EXCLUDED.updated_at, + content = EXCLUDED.content, + embedding = EXCLUDED.embedding, + language = EXCLUDED.language, + path = EXCLUDED.path, + start_line = EXCLUDED.start_line, + end_line = EXCLUDED.end_line, + commit_sha = EXCLUDED.commit_sha + """; + + private final JdbcClient jdbcClient; + private final JdbcTemplate jdbcTemplate; + + private static void bindChunk(PreparedStatement statement, CodeChunk chunk, Timestamp now) + throws SQLException { + UUID id = chunk.getId(); + if (id == null) { + id = UUID.randomUUID(); + chunk.setId(id); + } + + statement.setObject(1, id); + statement.setTimestamp(2, now); + statement.setTimestamp(3, now); + statement.setObject(4, requiredId(chunk.getFile(), "file")); + statement.setObject(5, requiredId(chunk.getCodebase(), "codebase")); + statement.setObject(6, chunk.getChunkIndex()); + statement.setString(7, chunk.getContent()); + setVector(statement, 8, chunk.getEmbedding()); + statement.setString(9, chunk.getLanguage()); + statement.setString(10, chunk.getPath()); + setNullableInt(statement, 11, chunk.getStartLine()); + setNullableInt(statement, 12, chunk.getEndLine()); + statement.setString(13, chunk.getCommitSha()); + } + + private static void setVector(PreparedStatement statement, int index, float[] vector) + throws SQLException { + if (vector == null) { + statement.setNull(index, Types.OTHER); + } else { + statement.setObject(index, new PGvector(vector)); + } + } + + private static void setNullableInt(PreparedStatement statement, int index, Integer value) + throws SQLException { + if (value == null) { + statement.setNull(index, Types.INTEGER); + } else { + statement.setInt(index, value); + } + } + + private static UUID requiredId(Object entity, String name) { + UUID id = entity instanceof RepositoryFile file ? file.getId() + : entity instanceof Codebase codebase ? codebase.getId() : null; + if (id == null) { + throw new IllegalArgumentException("A code chunk must have a " + name + " id"); + } + return id; + } + + private static boolean hasText(String value) { + return value != null && !value.isBlank(); + } + + private static float[] readVector(Object value) throws SQLException { + if (value == null) { + return null; + } + if (value instanceof PGvector vector) { + return vector.toArray(); + } + + String literal = value instanceof PGobject pgObject ? pgObject.getValue() : value.toString(); + return new PGvector(literal).toArray(); + } + + private static Instant readInstant(ResultSet rs, String column) throws SQLException { + Timestamp timestamp = rs.getTimestamp(column); + return timestamp == null ? null : timestamp.toInstant(); + } + + @Override + public void saveAll(Collection chunks) { + if (chunks == null || chunks.isEmpty()) { + return; + } + + Timestamp now = Timestamp.from(Instant.now()); + jdbcTemplate.batchUpdate(INSERT_SQL, chunks, BATCH_SIZE, + (statement, chunk) -> bindChunk(statement, chunk, now)); + var chunksByFile = chunks.stream() + .collect(java.util.stream.Collectors.groupingBy( + chunk -> requiredId(chunk.getFile(), "file"))); + for (var entry : chunksByFile.entrySet()) { + var chunkIndexes = entry.getValue().stream() + .map(CodeChunk::getChunkIndex) + .toList(); + Map persistedIds = new HashMap<>(); + jdbcClient.sql(""" + SELECT id, chunk_index FROM code_chunks + WHERE file_id = :fileId AND chunk_index IN (:chunkIndexes) + """) + .param("fileId", entry.getKey()) + .param("chunkIndexes", chunkIndexes) + .query((rs, rowNum) -> Map.entry( + rs.getInt("chunk_index"), rs.getObject("id", UUID.class))) + .list() + .forEach(id -> persistedIds.put(id.getKey(), id.getValue())); + for (CodeChunk chunk : entry.getValue()) { + UUID persistedId = persistedIds.get(chunk.getChunkIndex()); + if (persistedId == null) { + throw new IllegalStateException("Persisted code chunk was not found"); + } + chunk.setId(persistedId); + } + } + } + + @Override + public void updateEmbedding(UUID chunkId, float[] embedding) { + jdbcClient.sql(""" + UPDATE code_chunks + SET embedding = :embedding, updated_at = :updatedAt + WHERE id = :id + """) + .param("embedding", embedding == null ? null : new PGvector(embedding)) + .param("updatedAt", Timestamp.from(Instant.now())) + .param("id", chunkId) + .update(); + } + + @Override + public void deleteByFileId(UUID fileId) { + jdbcClient.sql("DELETE FROM code_chunks WHERE file_id = :fileId") + .param("fileId", fileId) + .update(); + } + + @Override + public void deleteByCodebaseId(UUID codebaseId) { + jdbcClient.sql("DELETE FROM code_chunks WHERE codebase_id = :codebaseId") + .param("codebaseId", codebaseId) + .update(); + } + + @Override + public Optional findById(UUID chunkId) { + return jdbcClient.sql(""" + SELECT %s + FROM code_chunks c + WHERE c.id = :id + """.formatted(CHUNK_COLUMNS)) + .param("id", chunkId) + .query(this::mapChunk) + .optional(); + } + + @Override + public List similaritySearch(SimilaritySearchRequest request) { + if (request == null || request.codebaseId() == null + || request.embedding() == null || request.embedding().length == 0) { + return List.of(); + } + + StringBuilder sql = new StringBuilder(""" + SELECT ranked.* + FROM ( + SELECT %s, c.embedding <=> :embedding AS distance + FROM code_chunks c + """.formatted(CHUNK_COLUMNS)); + + if (hasText(request.branch())) { + sql.append(" JOIN codebases b ON b.id = c.codebase_id"); + } + sql.append(""" + WHERE c.codebase_id = :codebaseId + AND c.embedding IS NOT NULL + """); + if (hasText(request.language())) { + sql.append(" AND c.language = :language"); + } + if (hasText(request.branch())) { + sql.append(" AND b.branch = :branch"); + } + if (hasText(request.commitSha())) { + sql.append(" AND c.commit_sha = :commitSha"); + } + sql.append(""" + ) ranked + WHERE (:maxDistance IS NULL OR ranked.distance <= :maxDistance) + ORDER BY ranked.distance + LIMIT :topK + """); + + var statement = jdbcClient.sql(sql.toString()) + .param("codebaseId", request.codebaseId()) + .param("embedding", new PGvector(request.embedding())) + .param("maxDistance", request.maxDistance()) + .param("topK", Math.max(1, request.topK())); + if (hasText(request.language())) { + statement = statement.param("language", request.language()); + } + if (hasText(request.branch())) { + statement = statement.param("branch", request.branch()); + } + if (hasText(request.commitSha())) { + statement = statement.param("commitSha", request.commitSha()); + } + + return statement.query(this::mapSimilarityResult).list(); + } + + private SimilaritySearchResult mapSimilarityResult(ResultSet rs, int rowNum) throws SQLException { + return new SimilaritySearchResult(mapChunk(rs, rowNum), rs.getDouble("distance")); + } + + @Override + public long countByCodebaseId(UUID codebaseId) { + return jdbcClient.sql("SELECT COUNT(*) FROM code_chunks WHERE codebase_id = :codebaseId") + .param("codebaseId", codebaseId) + .query(Long.class) + .single(); + } + + private CodeChunk mapChunk(ResultSet rs, int rowNum) throws SQLException { + CodeChunk chunk = CodeChunk.builder() + .id(rs.getObject("id", UUID.class)) + .file(RepositoryFile.builder().id(rs.getObject("file_id", UUID.class)).build()) + .codebase(Codebase.builder().id(rs.getObject("codebase_id", UUID.class)).build()) + .chunkIndex((Integer) rs.getObject("chunk_index")) + .content(rs.getString("content")) + .embedding(readVector(rs.getObject("embedding"))) + .language(rs.getString("language")) + .path(rs.getString("path")) + .startLine((Integer) rs.getObject("start_line")) + .endLine((Integer) rs.getObject("end_line")) + .commitSha(rs.getString("commit_sha")) + .build(); + chunk.setCreatedAt(readInstant(rs, "created_at")); + chunk.setUpdatedAt(readInstant(rs, "updated_at")); + return chunk; + } +} diff --git a/server/src/main/java/com/meet/server/feature/codechunk/SimilaritySearchResult.java b/server/src/main/java/com/meet/server/feature/codechunk/SimilaritySearchResult.java new file mode 100644 index 0000000..6e5c1b7 --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/codechunk/SimilaritySearchResult.java @@ -0,0 +1,4 @@ +package com.meet.server.feature.codechunk; + +public record SimilaritySearchResult(CodeChunk chunk, double distance) { +} diff --git a/server/src/main/java/com/meet/server/feature/embedding/EmbeddingService.java b/server/src/main/java/com/meet/server/feature/embedding/EmbeddingService.java new file mode 100644 index 0000000..0abff06 --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/embedding/EmbeddingService.java @@ -0,0 +1,34 @@ +package com.meet.server.feature.embedding; + +import com.meet.server.feature.codechunk.CodeChunk; +import com.meet.server.feature.codechunk.CodeChunkRepository; +import lombok.RequiredArgsConstructor; +import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +@RequiredArgsConstructor +public class EmbeddingService { + + private static final int EMBEDDING_BATCH_SIZE = 32; + + private final CodeChunkRepository codeChunkRepository; + private final EmbeddingModel embeddingModel; + + public void embedChunks(List chunks) { + for (int start = 0; start < chunks.size(); start += EMBEDDING_BATCH_SIZE) { + int end = Math.min(start + EMBEDDING_BATCH_SIZE, chunks.size()); + var batch = chunks.subList(start, end); + var embeddings = embeddingModel.embed(batch.stream() + .map(CodeChunk::getContent) + .toList()); + for (int index = 0; index < batch.size(); index++) { + batch.get(index).setEmbedding(embeddings.get(index)); + } + } + + codeChunkRepository.saveAll(chunks); + } +} diff --git a/server/src/main/java/com/meet/server/feature/embedding/SimilaritySearchRequest.java b/server/src/main/java/com/meet/server/feature/embedding/SimilaritySearchRequest.java new file mode 100644 index 0000000..e238ef9 --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/embedding/SimilaritySearchRequest.java @@ -0,0 +1,14 @@ +package com.meet.server.feature.embedding; + +import java.util.UUID; + +public record SimilaritySearchRequest( + UUID codebaseId, + float[] embedding, + int topK, + Double maxDistance, + String language, + String branch, + String commitSha +) { +} diff --git a/server/src/main/java/com/meet/server/feature/indexing/extractor/ChunkExtractor.java b/server/src/main/java/com/meet/server/feature/indexing/extractor/ChunkExtractor.java new file mode 100644 index 0000000..50279e4 --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/indexing/extractor/ChunkExtractor.java @@ -0,0 +1,15 @@ +package com.meet.server.feature.indexing.extractor; + +import com.meet.server.feature.codechunk.CodeChunk; +import com.meet.server.feature.indexing.language.Language; +import com.meet.server.feature.indexing.parser.ParsedFile; + +import java.util.List; + +public interface ChunkExtractor { + + boolean supports(Language language); + + List extract(ParsedFile parsed); + +} diff --git a/server/src/main/java/com/meet/server/feature/indexing/extractor/CssExtractor.java b/server/src/main/java/com/meet/server/feature/indexing/extractor/CssExtractor.java new file mode 100644 index 0000000..b771277 --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/indexing/extractor/CssExtractor.java @@ -0,0 +1,97 @@ +package com.meet.server.feature.indexing.extractor; + +import com.meet.server.feature.codechunk.CodeChunk; +import com.meet.server.feature.indexing.language.Language; +import com.meet.server.feature.indexing.parser.ParsedFile; +import org.springframework.stereotype.Component; +import org.treesitter.TSNode; + +import java.util.ArrayList; +import java.util.List; + +@Component +public class CssExtractor implements ChunkExtractor { + + @Override + public boolean supports(Language language) { + return language == Language.CSS; + } + + @Override + public List extract(ParsedFile parsed) { + if (parsed.rootNode() == null) { + return List.of(); + } + var chunks = new ArrayList(); + for (int i = 0; i < parsed.rootNode().getNamedChildCount(); i++) { + emit(parsed, parsed.rootNode().getNamedChild(i), "", chunks); + } + return chunks; + } + + private void emit(ParsedFile parsed, TSNode node, String inheritedContext, List chunks) { + String content = TreeSitterChunkSupport.source(parsed, node); + if (content.isBlank() || "comment".equals(node.getType())) { + return; + } + String complete = inheritedContext + content; + if (complete.length() <= TreeSitterChunkSupport.MAX_CHUNK_CHARACTERS) { + TreeSitterChunkSupport.addChunk(parsed, node, chunks, complete); + return; + } + + TSNode block = blockChild(node); + String context = block == null + ? TreeSitterChunkSupport.boundedContext(inheritedContext) + : TreeSitterChunkSupport.boundedContext(inheritedContext + + TreeSitterChunkSupport.source(parsed, node.getStartByte(), block.getStartByte())); + int before = chunks.size(); + if (block != null) { + for (int i = 0; i < block.getNamedChildCount(); i++) { + emit(parsed, block.getNamedChild(i), context, chunks); + } + } + if (chunks.size() == before) { + for (int i = 0; i < node.getNamedChildCount(); i++) { + TSNode child = node.getNamedChild(i); + if (child != block) { + emit(parsed, child, context, chunks); + } + } + } + if (chunks.size() == before) { + emitLineChunks(parsed, node, inheritedContext, chunks); + } + } + + private TSNode blockChild(TSNode node) { + for (int i = 0; i < node.getNamedChildCount(); i++) { + TSNode child = node.getNamedChild(i); + if (switch (child.getType()) { + case "block", "declaration_block", "keyframes_block" -> true; + default -> false; + }) { + return child; + } + } + return null; + } + + private void emitLineChunks(ParsedFile parsed, TSNode node, String context, List chunks) { + String[] lines = TreeSitterChunkSupport.source(parsed, node).split("\\R", -1); + String prefix = TreeSitterChunkSupport.boundedContext(context); + StringBuilder current = new StringBuilder(prefix); + for (String line : lines) { + if (current.length() + line.length() + 1 > TreeSitterChunkSupport.MAX_CHUNK_CHARACTERS + && current.length() > prefix.length()) { + TreeSitterChunkSupport.addChunk(parsed, node, chunks, current.toString()); + current = new StringBuilder(prefix); + } + if (current.length() > prefix.length()) { + current.append('\n'); + } + current.append(line); + } + TreeSitterChunkSupport.addChunk(parsed, node, chunks, current.toString()); + } +} diff --git a/server/src/main/java/com/meet/server/feature/indexing/extractor/HtmlExtractor.java b/server/src/main/java/com/meet/server/feature/indexing/extractor/HtmlExtractor.java new file mode 100644 index 0000000..ba7c776 --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/indexing/extractor/HtmlExtractor.java @@ -0,0 +1,101 @@ +package com.meet.server.feature.indexing.extractor; + +import com.meet.server.feature.codechunk.CodeChunk; +import com.meet.server.feature.indexing.language.Language; +import com.meet.server.feature.indexing.parser.ParsedFile; +import org.springframework.stereotype.Component; +import org.treesitter.TSNode; + +import java.util.ArrayList; +import java.util.List; + +@Component +public class HtmlExtractor implements ChunkExtractor { + + private static final int MAX_RECURSION_DEPTH = 64; + + @Override + public boolean supports(Language language) { + return language == Language.HTML; + } + + @Override + public List extract(ParsedFile parsed) { + if (parsed.rootNode() == null) { + return List.of(); + } + var chunks = new ArrayList(); + for (int i = 0; i < parsed.rootNode().getNamedChildCount(); i++) { + emit(parsed, parsed.rootNode().getNamedChild(i), "", 0, chunks); + } + return chunks; + } + + private void emit(ParsedFile parsed, TSNode node, String inheritedContext, int depth, + List chunks) { + String content = TreeSitterChunkSupport.source(parsed, node); + if (content.isBlank() || "comment".equals(node.getType())) { + return; + } + String complete = inheritedContext + content; + if (complete.length() <= TreeSitterChunkSupport.MAX_CHUNK_CHARACTERS) { + TreeSitterChunkSupport.addChunk(parsed, node, chunks, complete); + return; + } + if (depth >= MAX_RECURSION_DEPTH) { + emitLineChunks(parsed, node, inheritedContext, chunks); + return; + } + + String context = TreeSitterChunkSupport.boundedContext(inheritedContext + openingContext(parsed, node)); + int before = chunks.size(); + for (int i = 0; i < node.getNamedChildCount(); i++) { + TSNode child = node.getNamedChild(i); + if (isStructuralWrapper(child) || "comment".equals(child.getType())) { + continue; + } + emit(parsed, child, context, depth + 1, chunks); + } + if (chunks.size() == before) { + emitLineChunks(parsed, node, inheritedContext, chunks); + } + } + + private String openingContext(ParsedFile parsed, TSNode node) { + if (node.getNamedChildCount() == 0) { + return ""; + } + TSNode first = node.getNamedChild(0); + return switch (first.getType()) { + case "start_tag", "script_start_tag", "style_start_tag" -> + TreeSitterChunkSupport.source(parsed, node.getStartByte(), first.getEndByte()); + default -> ""; + }; + } + + private boolean isStructuralWrapper(TSNode node) { + return switch (node.getType()) { + case "start_tag", "end_tag", "script_start_tag", "script_end_tag", + "style_start_tag", "style_end_tag" -> true; + default -> false; + }; + } + + private void emitLineChunks(ParsedFile parsed, TSNode node, String context, List chunks) { + String[] lines = TreeSitterChunkSupport.source(parsed, node).split("\\R", -1); + String prefix = TreeSitterChunkSupport.boundedContext(context); + StringBuilder current = new StringBuilder(prefix); + for (String line : lines) { + if (current.length() + line.length() + 1 > TreeSitterChunkSupport.MAX_CHUNK_CHARACTERS + && current.length() > prefix.length()) { + TreeSitterChunkSupport.addChunk(parsed, node, chunks, current.toString()); + current = new StringBuilder(prefix); + } + if (current.length() > prefix.length()) { + current.append('\n'); + } + current.append(line); + } + TreeSitterChunkSupport.addChunk(parsed, node, chunks, current.toString()); + } +} diff --git a/server/src/main/java/com/meet/server/feature/indexing/extractor/JsonExtractor.java b/server/src/main/java/com/meet/server/feature/indexing/extractor/JsonExtractor.java new file mode 100644 index 0000000..370fc17 --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/indexing/extractor/JsonExtractor.java @@ -0,0 +1,77 @@ +package com.meet.server.feature.indexing.extractor; + +import com.meet.server.feature.codechunk.CodeChunk; +import com.meet.server.feature.indexing.language.Language; +import com.meet.server.feature.indexing.parser.ParsedFile; +import org.springframework.stereotype.Component; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.json.JsonMapper; + +import java.util.ArrayList; +import java.util.List; + +@Component +public class JsonExtractor implements ChunkExtractor { + + private static final int MAX_CHUNK_CHARACTERS = 4_000; + private final JsonMapper jsonMapper; + + public JsonExtractor(JsonMapper jsonMapper) { + this.jsonMapper = jsonMapper; + } + + @Override + public boolean supports(Language language) { + return language == Language.JSON; + } + + @Override + public List extract(ParsedFile parsed) { + if (parsed.jsonTree() == null) { + return List.of(); + } + + var chunks = new ArrayList(); + emit(parsed, parsed.jsonTree(), "$", chunks); + return chunks; + } + + private void emit(ParsedFile parsed, JsonNode node, String path, List chunks) { + String serialized = jsonMapper.writeValueAsString(node); + String content = "JSON path: " + path + "\n" + serialized; + if (content.length() <= MAX_CHUNK_CHARACTERS || node.isValueNode()) { + emitBounded(parsed, path, content, chunks); + return; + } + + if (node.isObject()) { + for (var field : node.properties()) { + emit(parsed, field.getValue(), path + "." + field.getKey(), chunks); + } + } else if (node.isArray()) { + for (int index = 0; index < node.size(); index++) { + emit(parsed, node.get(index), path + "[" + index + "]", chunks); + } + } + } + + private void emitBounded(ParsedFile parsed, String path, String content, List chunks) { + for (int start = 0; start < content.length(); start += MAX_CHUNK_CHARACTERS) { + int end = Math.min(start + MAX_CHUNK_CHARACTERS, content.length()); + addChunk(parsed, path, content.substring(start, end), chunks); + } + } + + private void addChunk(ParsedFile parsed, String path, String content, List chunks) { + chunks.add(CodeChunk.builder() + .file(parsed.file()) + .codebase(parsed.file().getCodebase()) + .chunkIndex(chunks.size()) + .content(content) + .language(parsed.file().getLanguage()) + .path(parsed.file().getPath()) + .startLine(null) + .endLine(null) + .build()); + } +} diff --git a/server/src/main/java/com/meet/server/feature/indexing/extractor/MarkdownExtractor.java b/server/src/main/java/com/meet/server/feature/indexing/extractor/MarkdownExtractor.java new file mode 100644 index 0000000..6ad49de --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/indexing/extractor/MarkdownExtractor.java @@ -0,0 +1,46 @@ +package com.meet.server.feature.indexing.extractor; + +import com.meet.server.feature.codechunk.CodeChunk; +import com.meet.server.feature.indexing.language.Language; +import com.meet.server.feature.indexing.parser.ParsedFile; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; + +@Component +public class MarkdownExtractor implements ChunkExtractor { + @Override + public boolean supports(Language language) { + return language == Language.MARKDOWN; + } + + @Override + public List extract(ParsedFile parsed) { + var chunks = new ArrayList(); + var current = new StringBuilder(); + int startLine = 1; + boolean inFence = false; + var lines = parsed.content().split("\\R", -1); + for (int i = 0; i < lines.length; i++) { + String line = lines[i]; + if (!inFence && line.startsWith("#") && !current.isEmpty()) { + add(parsed, chunks, current, startLine, i); + current.setLength(0); + startLine = i + 1; + } + current.append(line).append('\n'); + if (line.stripLeading().startsWith("```")) { + inFence = !inFence; + } + } + if (!current.toString().isBlank()) add(parsed, chunks, current, startLine, lines.length); + return chunks; + } + + private void add(ParsedFile parsed, List chunks, StringBuilder content, int start, int end) { + chunks.add(CodeChunk.builder().file(parsed.file()).codebase(parsed.file().getCodebase()) + .chunkIndex(chunks.size()).content(content.toString().stripTrailing()) + .language("markdown").path(parsed.file().getPath()).startLine(start).endLine(end).build()); + } +} diff --git a/server/src/main/java/com/meet/server/feature/indexing/extractor/TextExtractor.java b/server/src/main/java/com/meet/server/feature/indexing/extractor/TextExtractor.java new file mode 100644 index 0000000..eeac7ae --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/indexing/extractor/TextExtractor.java @@ -0,0 +1,44 @@ +package com.meet.server.feature.indexing.extractor; + +import com.meet.server.feature.codechunk.CodeChunk; +import com.meet.server.feature.indexing.language.Language; +import com.meet.server.feature.indexing.parser.ParsedFile; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** Creates bounded line-based chunks for text files without a syntax grammar. */ +@Component +public class TextExtractor implements ChunkExtractor { + private static final int MAX_LINES_PER_CHUNK = 120; + + @Override + public boolean supports(Language language) { + return language == Language.UNKNOWN; + } + + @Override + public List extract(ParsedFile parsed) { + var chunks = new ArrayList(); + var lines = parsed.content().split("\\R", -1); + for (int start = 0; start < lines.length; start += MAX_LINES_PER_CHUNK) { + int end = Math.min(start + MAX_LINES_PER_CHUNK, lines.length); + var content = String.join("\n", Arrays.asList(lines).subList(start, end)).stripTrailing(); + if (!content.isBlank()) { + chunks.add(CodeChunk.builder() + .file(parsed.file()) + .codebase(parsed.file().getCodebase()) + .chunkIndex(chunks.size()) + .content(content) + .language(parsed.file().getLanguage()) + .path(parsed.file().getPath()) + .startLine(start + 1) + .endLine(end) + .build()); + } + } + return chunks; + } +} diff --git a/server/src/main/java/com/meet/server/feature/indexing/extractor/TreeSitterChunkSupport.java b/server/src/main/java/com/meet/server/feature/indexing/extractor/TreeSitterChunkSupport.java new file mode 100644 index 0000000..f7fa714 --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/indexing/extractor/TreeSitterChunkSupport.java @@ -0,0 +1,52 @@ +package com.meet.server.feature.indexing.extractor; + +import com.meet.server.feature.codechunk.CodeChunk; +import com.meet.server.feature.indexing.parser.ParsedFile; +import org.treesitter.TSNode; + +import java.nio.charset.StandardCharsets; +import java.util.List; + +final class TreeSitterChunkSupport { + + static final int MAX_CHUNK_CHARACTERS = 4_000; + static final int MAX_CONTEXT_CHARACTERS = 1_200; + + private TreeSitterChunkSupport() { + } + + static String source(ParsedFile parsed, TSNode node) { + return source(parsed, node.getStartByte(), node.getEndByte()); + } + + static String source(ParsedFile parsed, int startByte, int endByte) { + byte[] bytes = parsed.content().getBytes(StandardCharsets.UTF_8); + int start = Math.max(0, Math.min(startByte, bytes.length)); + int end = Math.max(start, Math.min(endByte, bytes.length)); + return new String(bytes, start, end - start, StandardCharsets.UTF_8); + } + + static void addChunk(ParsedFile parsed, TSNode node, List chunks, String content) { + if (content == null || content.isBlank()) { + return; + } + chunks.add(CodeChunk.builder() + .file(parsed.file()) + .codebase(parsed.file().getCodebase()) + .chunkIndex(chunks.size()) + .content(content.stripTrailing()) + .language(parsed.file().getLanguage()) + .path(parsed.file().getPath()) + .startLine(node.getStartPoint().getRow() + 1) + .endLine(node.getEndPoint().getRow() + 1) + .build()); + } + + static String boundedContext(String context) { + if (context == null || context.isBlank()) { + return ""; + } + return context.substring(0, Math.min(context.length(), MAX_CONTEXT_CHARACTERS)).stripTrailing() + + "\n\n"; + } +} diff --git a/server/src/main/java/com/meet/server/feature/indexing/extractor/TreeSitterExtractor.java b/server/src/main/java/com/meet/server/feature/indexing/extractor/TreeSitterExtractor.java new file mode 100644 index 0000000..3b1e9c9 --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/indexing/extractor/TreeSitterExtractor.java @@ -0,0 +1,175 @@ +package com.meet.server.feature.indexing.extractor; + +import com.meet.server.feature.codechunk.CodeChunk; +import com.meet.server.feature.indexing.language.Language; +import com.meet.server.feature.indexing.parser.ParsedFile; +import org.springframework.stereotype.Component; +import org.treesitter.TSNode; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +@Component +public class TreeSitterExtractor implements ChunkExtractor { + + private static final int MAX_CHUNK_CHARACTERS = 4_000; + private static final int MAX_CONTEXT_CHARACTERS = 1_200; + + @Override + public boolean supports(Language language) { + return language.isProgramming(); + } + + @Override + public List extract(ParsedFile parsed) { + if (parsed.rootNode() == null) { + return List.of(); + } + + var chunks = new ArrayList(); + for (int i = 0; i < parsed.rootNode().getNamedChildCount(); i++) { + emitTopLevel(parsed, parsed.rootNode().getNamedChild(i), chunks); + } + return chunks; + } + + private void emitTopLevel(ParsedFile parsed, TSNode node, List chunks) { + if (parsed.language().isIgnoredNode(node.getType())) { + return; + } + + if (sourceLength(parsed, node) <= MAX_CHUNK_CHARACTERS) { + addChunk(parsed, node, chunks, source(parsed, node)); + return; + } + + if (parsed.language().isTypeDeclaration(node.getType())) { + emitLargeType(parsed, node, chunks); + } else { + emitLineChunks(parsed, node, chunks, ""); + } + } + + private void emitLargeType(ParsedFile parsed, TSNode type, List chunks) { + TSNode body = type.getChildByFieldName("body"); + if (body == null || body.isNull() || body.getNamedChildCount() == 0) { + emitLineChunks(parsed, type, chunks, ""); + return; + } + + String context = typeContext(parsed, type, body); + int before = chunks.size(); + for (int i = 0; i < body.getNamedChildCount(); i++) { + TSNode member = body.getNamedChild(i); + if (parsed.language().isIgnoredNode(member.getType()) + || !parsed.language().isMemberDeclaration(member.getType())) { + continue; + } + emitMember(parsed, member, context, chunks); + } + + if (chunks.size() == before) { + emitLineChunks(parsed, type, chunks, context); + } + } + + private void emitMember(ParsedFile parsed, TSNode member, String context, List chunks) { + String content = context + source(parsed, member); + if (content.length() <= MAX_CHUNK_CHARACTERS) { + addChunk(parsed, member, chunks, content); + } else if (parsed.language().isTypeDeclaration(member.getType())) { + emitLargeType(parsed, member, chunks); + } else { + emitLineChunks(parsed, member, chunks, context); + } + } + + private String typeContext(ParsedFile parsed, TSNode type, TSNode body) { + String signature = source(parsed, type.getStartByte(), body.getStartByte()).strip(); + var fields = new StringBuilder(); + for (int i = 0; i < body.getNamedChildCount(); i++) { + TSNode child = body.getNamedChild(i); + if (!parsed.language().isFieldDeclaration(child.getType())) { + continue; + } + String field = source(parsed, child).strip(); + if (fields.length() + field.length() + 1 > MAX_CONTEXT_CHARACTERS / 2) { + break; + } + fields.append(field).append('\n'); + } + + String context = "Class context:\n" + signature; + if (!fields.isEmpty()) { + context += "\nFields:\n" + fields.toString().stripTrailing(); + } + return context.substring(0, Math.min(context.length(), MAX_CONTEXT_CHARACTERS)) + + "\n\nMember:\n"; + } + + private void emitLineChunks(ParsedFile parsed, TSNode node, List chunks, String context) { + String[] lines = source(parsed, node).split("\\R", -1); + StringBuilder current = new StringBuilder(context); + int chunkStartLine = node.getStartPoint().getRow() + 1; + int currentStartLine = chunkStartLine; + + for (int index = 0; index < lines.length; index++) { + String line = lines[index]; + int requiredLength = current.length() + line.length() + (current.length() > context.length() ? 1 : 0); + if (requiredLength > MAX_CHUNK_CHARACTERS && current.length() > context.length()) { + addChunk(parsed, node, chunks, current.toString().stripTrailing(), + currentStartLine, chunkStartLine + index - 1); + current = new StringBuilder(context); + currentStartLine = chunkStartLine + index; + } + if (current.length() > context.length()) { + current.append('\n'); + } + current.append(line); + } + + if (current.length() > context.length() && !current.toString().isBlank()) { + addChunk(parsed, node, chunks, current.toString().stripTrailing(), + currentStartLine, node.getEndPoint().getRow() + 1); + } + } + + private int sourceLength(ParsedFile parsed, TSNode node) { + return source(parsed, node).length(); + } + + private String source(ParsedFile parsed, TSNode node) { + return source(parsed, node.getStartByte(), node.getEndByte()); + } + + private String source(ParsedFile parsed, int startByte, int endByte) { + byte[] bytes = parsed.content().getBytes(StandardCharsets.UTF_8); + int start = Math.max(0, Math.min(startByte, bytes.length)); + int end = Math.max(start, Math.min(endByte, bytes.length)); + return new String(bytes, start, end - start, StandardCharsets.UTF_8); + } + + private void addChunk(ParsedFile parsed, TSNode node, List chunks, String content) { + addChunk(parsed, node, chunks, content, + node.getStartPoint().getRow() + 1, + node.getEndPoint().getRow() + 1); + } + + private void addChunk(ParsedFile parsed, TSNode node, List chunks, + String content, int startLine, int endLine) { + if (content == null || content.isBlank()) { + return; + } + chunks.add(CodeChunk.builder() + .file(parsed.file()) + .codebase(parsed.file().getCodebase()) + .chunkIndex(chunks.size()) + .content(content) + .language(parsed.file().getLanguage()) + .path(parsed.file().getPath()) + .startLine(startLine) + .endLine(endLine) + .build()); + } +} diff --git a/server/src/main/java/com/meet/server/feature/indexing/language/Language.java b/server/src/main/java/com/meet/server/feature/indexing/language/Language.java new file mode 100644 index 0000000..ae7e767 --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/indexing/language/Language.java @@ -0,0 +1,137 @@ +package com.meet.server.feature.indexing.language; + +import lombok.Getter; + +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +public enum Language { + JAVA(true, ParserKind.TREE_SITTER, "TreeSitterJava", null, new String[]{"java"}), + KOTLIN(true, ParserKind.TREE_SITTER, "TreeSitterKotlin", null, new String[]{"kotlin", "kt", "kts"}), + PYTHON(true, ParserKind.TREE_SITTER, "TreeSitterPython", null, new String[]{"python", "py"}), + JAVASCRIPT(true, ParserKind.TREE_SITTER, "TreeSitterJavascript", null, new String[]{"javascript", "js", "jsx", "mjs", "cjs"}), + TYPESCRIPT(true, ParserKind.TREE_SITTER, "TreeSitterTypescript", null, new String[]{"typescript", "ts"}), + TSX(true, ParserKind.TREE_SITTER, "TreeSitterTsx", null, new String[]{"tsx"}), + GO(true, ParserKind.TREE_SITTER, "TreeSitterGo", null, new String[]{"go"}), + RUST(true, ParserKind.TREE_SITTER, "TreeSitterRust", null, new String[]{"rust", "rs"}), + C(true, ParserKind.TREE_SITTER, "TreeSitterC", null, new String[]{"c"}), + CPP(true, ParserKind.TREE_SITTER, "TreeSitterCpp", null, new String[]{"cpp", "c++", "cc", "cxx", "hpp", "hxx"}), + CSHARP(true, ParserKind.TREE_SITTER, "TreeSitterCSharp", null, new String[]{"csharp", "c#", "cs"}), + PHP(true, ParserKind.TREE_SITTER, "TreeSitterPhp", null, new String[]{"php"}), + RUBY(true, ParserKind.TREE_SITTER, "TreeSitterRuby", null, new String[]{"ruby", "rb"}), + SWIFT(true, ParserKind.TREE_SITTER, "TreeSitterSwift", null, new String[]{"swift"}), + + HTML(false, ParserKind.TREE_SITTER, "TreeSitterHtml", null, new String[]{"html", "htm"}), + CSS(false, ParserKind.TREE_SITTER, "TreeSitterCss", null, new String[]{"css"}), + + SQL(false, ParserKind.TEXT, null, null, new String[]{"sql"}), + JSON(false, ParserKind.JSON, null, null, new String[]{"json"}), + YAML(false, ParserKind.TEXT, null, null, new String[]{"yaml", "yml"}), + XML(false, ParserKind.TEXT, null, null, new String[]{"xml"}), + MARKDOWN(false, ParserKind.MARKDOWN, null, null, new String[]{"markdown", "md"}), + DOCKERFILE(false, ParserKind.TEXT, null, null, new String[]{"dockerfile"}), + PROPERTIES(false, ParserKind.TEXT, null, null, new String[]{"properties", "props"}), + + UNKNOWN(false, ParserKind.TEXT, null, null, new String[0]); + + public enum ParserKind { + TREE_SITTER, JSON, MARKDOWN, TEXT + } + + private static final Set IGNORED_NODES = Set.of( + "comment", "package_declaration", "import_declaration", "import_statement", + "using_directive", "preproc_include", "preproc_def", "namespace_import" + ); + private static final Set TYPE_DECLARATIONS = Set.of( + "class_declaration", "interface_declaration", "enum_declaration", "record_declaration", + "annotation_type_declaration", "struct_declaration", "namespace_definition", + "object_declaration", "trait_item", "impl_item", "mod_item" + ); + private static final Set MEMBER_DECLARATIONS = Set.of( + "field_declaration", "method_declaration", "function_declaration", "function_definition", + "constructor_declaration", "compact_constructor_declaration", "static_initializer", + "initializer", "lexical_declaration", "variable_declaration", "property_declaration", + "const_item", "function_item", "struct_item", "enum_item", "class_declaration", + "interface_declaration", "enum_declaration", "record_declaration", "struct_declaration", + "object_declaration", "trait_item", "impl_item" + ); + private static final Set FIELD_DECLARATIONS = Set.of( + "field_declaration", "field_definition", "class_field", "property_declaration" + ); + private static final Map BY_ALIAS = buildAliasMap(); + + @Getter + private final boolean programming; + private final ParserKind parserKind; + private final String grammarClass; + private final String grammarMethod; + private final Set aliases; + + Language(boolean programming, ParserKind parserKind, String grammarClass, String grammarMethod, String[] aliases) { + this.programming = programming; + this.parserKind = parserKind; + this.grammarClass = grammarClass; + this.grammarMethod = grammarMethod; + this.aliases = Set.of(aliases); + } + + public static Language from(String value) { + if (value == null || value.isBlank()) return UNKNOWN; + String normalized = value.toLowerCase(Locale.ROOT).strip(); + if (normalized.startsWith(".")) normalized = normalized.substring(1); + return BY_ALIAS.getOrDefault(normalized, UNKNOWN); + } + + public static String extensionOf(String path) { + if (path == null || path.isBlank()) return null; + String fileName = Path.of(path).getFileName().toString(); + int dot = fileName.lastIndexOf('.'); + return dot > 0 ? fileName.substring(dot + 1).toLowerCase(Locale.ROOT) + : fileName.toLowerCase(Locale.ROOT); + } + + private static Map buildAliasMap() { + var aliases = new HashMap(); + for (Language language : values()) { + for (String alias : language.aliases) { + aliases.put(alias, language); + } + } + return Map.copyOf(aliases); + } + + public String grammarClass() { + return grammarClass; + } + + public String grammarMethod() { + return grammarMethod; + } + + public ParserKind parserKind() { + return parserKind; + } + + public boolean hasGrammar() { + return grammarClass != null; + } + + public boolean isIgnoredNode(String nodeType) { + return IGNORED_NODES.contains(nodeType); + } + + public boolean isTypeDeclaration(String nodeType) { + return TYPE_DECLARATIONS.contains(nodeType); + } + + public boolean isMemberDeclaration(String nodeType) { + return MEMBER_DECLARATIONS.contains(nodeType); + } + + public boolean isFieldDeclaration(String nodeType) { + return FIELD_DECLARATIONS.contains(nodeType); + } +} diff --git a/server/src/main/java/com/meet/server/feature/indexing/parser/JsonParser.java b/server/src/main/java/com/meet/server/feature/indexing/parser/JsonParser.java new file mode 100644 index 0000000..dd4996e --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/indexing/parser/JsonParser.java @@ -0,0 +1,40 @@ +package com.meet.server.feature.indexing.parser; + +import com.meet.server.feature.indexing.language.Language; +import com.meet.server.feature.repositoryfile.RepositoryFile; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import tools.jackson.core.JacksonException; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.json.JsonMapper; + +@Slf4j +@Component +public class JsonParser implements Parser { + + private final JsonMapper jsonMapper; + + public JsonParser(JsonMapper jsonMapper) { + this.jsonMapper = jsonMapper; + } + + @Override + public boolean supports(Language language) { + return language.parserKind() == Language.ParserKind.JSON; + } + + @Override + public ParsedFile parse(RepositoryFile file, String content) { + try { + JsonNode jsonTree = jsonMapper.readTree(content); + if (jsonTree == null) { + log.warn("Unable to parse empty JSON file {}; using text extraction", file.getPath()); + return new ParsedFile(file, Language.UNKNOWN, content, null, null, null); + } + return new ParsedFile(file, Language.JSON, content, null, null, jsonTree); + } catch (JacksonException exception) { + log.warn("Unable to parse JSON file {}; using text extraction", file.getPath()); + return new ParsedFile(file, Language.UNKNOWN, content, null, null, null); + } + } +} diff --git a/server/src/main/java/com/meet/server/feature/indexing/parser/MarkdownParser.java b/server/src/main/java/com/meet/server/feature/indexing/parser/MarkdownParser.java new file mode 100644 index 0000000..936109a --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/indexing/parser/MarkdownParser.java @@ -0,0 +1,19 @@ +package com.meet.server.feature.indexing.parser; + +import com.meet.server.feature.indexing.language.Language; +import com.meet.server.feature.repositoryfile.RepositoryFile; +import org.springframework.stereotype.Component; + +/** Keeps Markdown parsing independent from native Tree-sitter grammars for now. */ +@Component +public class MarkdownParser implements Parser { + @Override + public boolean supports(Language language) { + return language.parserKind() == Language.ParserKind.MARKDOWN; + } + + @Override + public ParsedFile parse(RepositoryFile file, String content) { + return new ParsedFile(file, Language.MARKDOWN, content, null, null, null); + } +} diff --git a/server/src/main/java/com/meet/server/feature/indexing/parser/ParsedFile.java b/server/src/main/java/com/meet/server/feature/indexing/parser/ParsedFile.java new file mode 100644 index 0000000..813b133 --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/indexing/parser/ParsedFile.java @@ -0,0 +1,21 @@ +package com.meet.server.feature.indexing.parser; + +import com.meet.server.feature.indexing.language.Language; +import com.meet.server.feature.repositoryfile.RepositoryFile; +import org.treesitter.TSNode; +import org.treesitter.TSTree; +import tools.jackson.databind.JsonNode; + + +public record ParsedFile( + RepositoryFile file, + Language language, + String content, + TSTree syntaxTree, + TSNode rootNode, + JsonNode jsonTree +) { + public boolean hasSyntaxErrors() { + return rootNode != null && rootNode.hasError(); + } +} diff --git a/server/src/main/java/com/meet/server/feature/indexing/parser/Parser.java b/server/src/main/java/com/meet/server/feature/indexing/parser/Parser.java new file mode 100644 index 0000000..056139c --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/indexing/parser/Parser.java @@ -0,0 +1,14 @@ +package com.meet.server.feature.indexing.parser; + +import com.meet.server.feature.indexing.language.Language; +import com.meet.server.feature.repositoryfile.RepositoryFile; + +public interface Parser { + + boolean supports(Language language); + + ParsedFile parse( + RepositoryFile file, + String content + ); +} diff --git a/server/src/main/java/com/meet/server/feature/indexing/parser/TextParser.java b/server/src/main/java/com/meet/server/feature/indexing/parser/TextParser.java new file mode 100644 index 0000000..86425f5 --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/indexing/parser/TextParser.java @@ -0,0 +1,18 @@ +package com.meet.server.feature.indexing.parser; + +import com.meet.server.feature.indexing.language.Language; +import com.meet.server.feature.repositoryfile.RepositoryFile; +import org.springframework.stereotype.Component; + +@Component +public class TextParser implements Parser { + @Override + public boolean supports(Language language) { + return language.parserKind() == Language.ParserKind.TEXT; + } + + @Override + public ParsedFile parse(RepositoryFile file, String content) { + return new ParsedFile(file, Language.UNKNOWN, content, null, null, null); + } +} diff --git a/server/src/main/java/com/meet/server/feature/indexing/parser/TreeSitterParser.java b/server/src/main/java/com/meet/server/feature/indexing/parser/TreeSitterParser.java new file mode 100644 index 0000000..4a30ab4 --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/indexing/parser/TreeSitterParser.java @@ -0,0 +1,57 @@ +package com.meet.server.feature.indexing.parser; + +import com.meet.server.feature.indexing.language.Language; +import com.meet.server.feature.repositoryfile.RepositoryFile; +import org.springframework.stereotype.Component; +import org.treesitter.TSLanguage; +import org.treesitter.TSParser; +import org.treesitter.TSTree; + +import java.lang.reflect.Method; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +@Component +public class TreeSitterParser implements Parser { + + private final Map languages = new ConcurrentHashMap<>(); + + @Override + public boolean supports(Language language) { + return language.parserKind() == Language.ParserKind.TREE_SITTER; + } + + @Override + public ParsedFile parse(RepositoryFile file, String content) { + var language = Language.from(file.getLanguage()); + var grammar = languages.computeIfAbsent(language, this::loadGrammar); + if (grammar == null) { + throw new IllegalArgumentException("No Tree-sitter grammar configured for " + language); + } + + var parser = new TSParser(); + if (!parser.setLanguage(grammar)) { + throw new IllegalStateException("Unable to configure Tree-sitter grammar for " + language); + } + TSTree tree = parser.parseString(null, content); + return new ParsedFile(file, language, content, tree, tree.getRootNode(), null); + } + + public boolean supports(String language) { + return Language.from(language).hasGrammar(); + } + + private TSLanguage loadGrammar(Language language) { + var className = "org.treesitter." + language.grammarClass(); + try { + Class grammarClass = Class.forName(className); + if (language.grammarMethod() != null) { + Method method = grammarClass.getMethod(language.grammarMethod()); + return (TSLanguage) method.invoke(null); + } + return (TSLanguage) grammarClass.getConstructor().newInstance(); + } catch (ReflectiveOperationException exception) { + throw new IllegalStateException("Tree-sitter grammar is not available: " + className, exception); + } + } +} diff --git a/server/src/main/java/com/meet/server/feature/repositoryfile/RepositoryFileDescriptor.java b/server/src/main/java/com/meet/server/feature/repositoryfile/RepositoryFileDescriptor.java new file mode 100644 index 0000000..11815a8 --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/repositoryfile/RepositoryFileDescriptor.java @@ -0,0 +1,9 @@ +package com.meet.server.feature.repositoryfile; + +public record RepositoryFileDescriptor( + String path, + String language, + long size, + String checksum +) { +} diff --git a/server/src/main/java/com/meet/server/feature/repositoryfile/RepositoryFileProcessor.java b/server/src/main/java/com/meet/server/feature/repositoryfile/RepositoryFileProcessor.java new file mode 100644 index 0000000..25ffddf --- /dev/null +++ b/server/src/main/java/com/meet/server/feature/repositoryfile/RepositoryFileProcessor.java @@ -0,0 +1,78 @@ +package com.meet.server.feature.repositoryfile; + +import com.meet.server.feature.codebase.Codebase; +import com.meet.server.feature.embedding.EmbeddingService; +import com.meet.server.feature.indexing.extractor.ChunkExtractor; +import com.meet.server.feature.indexing.language.Language; +import com.meet.server.feature.indexing.parser.ParsedFile; +import com.meet.server.feature.indexing.parser.Parser; +import lombok.RequiredArgsConstructor; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +@Service +@RequiredArgsConstructor +public class RepositoryFileProcessor { + + private static final Logger log = LogManager.getLogger(RepositoryFileProcessor.class); + + private final RepositoryFileRepository repositoryFileRepository; + private final EmbeddingService embeddingService; + private final List parsers; + private final List extractors; + + public int process(Codebase codebase, Path repositoryPath, List files, + String commitSha) { + log.debug("Processing {} files for codebase {}", files.size(), codebase.getId()); + files.forEach(file -> processFile(codebase, repositoryPath, file, commitSha)); + log.debug("Processed {} files for codebase {}", files.size(), codebase.getId()); + return files.size(); + } + + private void processFile(Codebase codebase, Path repositoryPath, RepositoryFileDescriptor descriptor, + String commitSha) { + if (isImageOrVideo(descriptor.path())) { + return; + } + + var repositoryFile = repositoryFileRepository.save(RepositoryFile.builder() + .codebase(codebase) + .path(descriptor.path()) + .language(descriptor.language()) + .checksum(descriptor.checksum()) + .size(descriptor.size()) + .build()); + try { + var content = Files.readString(repositoryPath.resolve(descriptor.path())); + var language = Language.from(descriptor.language()); + Parser parser = parsers.stream() + .filter(candidate -> candidate.supports(language)) + .findFirst() + .orElseThrow(() -> new IllegalStateException("No parser configured for " + language)); + ParsedFile parsed = parser.parse(repositoryFile, content); + var extractor = extractors.stream() + .filter(candidate -> candidate.supports(parsed.language())) + .findFirst() + .orElseThrow(() -> new IllegalStateException("No extractor configured for " + parsed.language())); + var chunks = extractor.extract(parsed); + chunks.forEach(chunk -> chunk.setCommitSha(commitSha)); + embeddingService.embedChunks(chunks); + } catch (IOException exception) { + throw new IllegalStateException("Unable to read repository file: " + descriptor.path(), exception); + } + } + + private boolean isImageOrVideo(String path) { + var lowerPath = path.toLowerCase(java.util.Locale.ROOT); + return java.util.Set.of( + ".apng", ".avif", ".bmp", ".gif", ".heic", ".jpeg", ".jpg", ".png", ".svg", ".webp", + ".avi", ".m4v", ".mkv", ".mov", ".mp4", ".mpeg", ".mpg", ".webm" + ).stream().anyMatch(lowerPath::endsWith); + } +} diff --git a/server/src/main/resources/application.yaml b/server/src/main/resources/application.yaml index 54bdf54..0a74a81 100644 --- a/server/src/main/resources/application.yaml +++ b/server/src/main/resources/application.yaml @@ -1,17 +1,23 @@ spring: application: name: server + ai: + model: + embedding: ollama + chat: ${CHAT_MODEL} + ollama: + embedding: + model: ${EMBEDDING_MODEL} + datasource: + url: ${POSTGRES_URL} + password: ${POSTGRES_PASSWORD} + username: ${POSTGRES_USER} data: redis: host: ${REDIS_HOST} port: ${REDIS_PORT} jpa: generate-ddl: off - datasource: - url: ${POSTGRES_URL} - password: ${POSTGRES_PASSWORD} - username: ${POSTGRES_USER} - security: oauth2: client: @@ -28,6 +34,9 @@ spring: scope: - read:user - user:email +logging: + level: + com.meet.server.feature.repositoryfile.RepositoryFileProcessor: DEBUG app: env: dev diff --git a/server/src/main/resources/db/migration/V3__create_repository_files_and_code_chunks.sql b/server/src/main/resources/db/migration/V3__create_repository_files_and_code_chunks.sql index b809e0a..682799a 100644 --- a/server/src/main/resources/db/migration/V3__create_repository_files_and_code_chunks.sql +++ b/server/src/main/resources/db/migration/V3__create_repository_files_and_code_chunks.sql @@ -2,14 +2,14 @@ CREATE EXTENSION IF NOT EXISTS vector; CREATE TABLE repository_files ( - id UUID NOT NULL, - created_at TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - updated_at TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - codebase_id UUID NOT NULL, - path VARCHAR(1024) NOT NULL, - language VARCHAR(255), - checksum VARCHAR(255), - size BIGINT, + id UUID NOT NULL, + created_at TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, + updated_at TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, + codebase_id UUID NOT NULL, + path VARCHAR(1024) NOT NULL, + language VARCHAR(255), + checksum VARCHAR(255), + size BIGINT, CONSTRAINT pk_repository_files PRIMARY KEY (id), CONSTRAINT uk_repository_files_codebase_path UNIQUE (codebase_id, path), CONSTRAINT uk_repository_files_id_codebase UNIQUE (id, codebase_id), @@ -20,19 +20,19 @@ CREATE INDEX idx_repository_files_codebase ON repository_files (codebase_id); CREATE TABLE code_chunks ( - id UUID NOT NULL, - created_at TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - updated_at TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, - file_id UUID NOT NULL, - codebase_id UUID NOT NULL, - chunk_index INTEGER NOT NULL, - content TEXT NOT NULL, - embedding vector, - language VARCHAR(255), - path VARCHAR(1024) NOT NULL, - start_line INTEGER, - end_line INTEGER, - commit_sha VARCHAR(255), + id UUID NOT NULL, + created_at TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, + updated_at TIMESTAMP(6) WITHOUT TIME ZONE NOT NULL, + file_id UUID NOT NULL, + codebase_id UUID NOT NULL, + chunk_index INTEGER NOT NULL, + content TEXT NOT NULL, + embedding vector(1024), + language VARCHAR(255), + path VARCHAR(1024) NOT NULL, + start_line INTEGER, + end_line INTEGER, + commit_sha VARCHAR(255), CONSTRAINT pk_code_chunks PRIMARY KEY (id), CONSTRAINT uk_code_chunks_file_chunk_index UNIQUE (file_id, chunk_index), CONSTRAINT fk_code_chunks_on_file_and_codebase FOREIGN KEY (file_id, codebase_id) diff --git a/server/src/main/resources/db/migration/V4__add_code_chunk_vector_index.sql b/server/src/main/resources/db/migration/V4__add_code_chunk_vector_index.sql new file mode 100644 index 0000000..a39f46b --- /dev/null +++ b/server/src/main/resources/db/migration/V4__add_code_chunk_vector_index.sql @@ -0,0 +1,3 @@ +CREATE INDEX idx_code_chunks_embedding_cosine + ON code_chunks USING hnsw (embedding vector_cosine_ops) + WHERE embedding IS NOT NULL; diff --git a/server/src/test/java/com/meet/server/feature/indexing/extractor/CssExtractorTest.java b/server/src/test/java/com/meet/server/feature/indexing/extractor/CssExtractorTest.java new file mode 100644 index 0000000..dd28f82 --- /dev/null +++ b/server/src/test/java/com/meet/server/feature/indexing/extractor/CssExtractorTest.java @@ -0,0 +1,68 @@ +package com.meet.server.feature.indexing.extractor; + +import com.meet.server.feature.codebase.Codebase; +import com.meet.server.feature.indexing.parser.ParsedFile; +import com.meet.server.feature.indexing.parser.TreeSitterParser; +import com.meet.server.feature.repositoryfile.RepositoryFile; +import org.junit.jupiter.api.Test; + +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.*; + +class CssExtractorTest { + + private final TreeSitterParser parser = new TreeSitterParser(); + private final CssExtractor extractor = new CssExtractor(); + + @Test + void emitsRulesAndPreservesSelectorContext() { + var chunks = extract(""" + @import url(\"fonts.css\"); + :root { --brand-color: #123456; } + .card, .panel { color: var(--brand-color); display: grid; } + @media (min-width: 800px) { .card { grid-template-columns: 1fr 2fr; } } + """); + + assertFalse(chunks.isEmpty()); + assertTrue(chunks.stream().anyMatch(chunk -> chunk.getContent().contains(".card"))); + assertTrue(chunks.stream().anyMatch(chunk -> chunk.getContent().contains("--brand-color"))); + assertTrue(chunks.stream().anyMatch(chunk -> chunk.getContent().contains("@media"))); + } + + @Test + void splitsLargeRuleIntoBoundedDeclarationChunks() { + var source = new StringBuilder(".large-component {\n"); + for (int i = 0; i < 400; i++) { + source.append(" --custom-property-").append(i).append(": ") + .append("x".repeat(20)).append(";\n"); + } + source.append("}\n"); + + var chunks = extract(source.toString()); + + assertTrue(chunks.size() > 1); + assertTrue(chunks.stream().allMatch(chunk -> chunk.getContent().length() <= 4_000)); + assertTrue(chunks.stream().allMatch(chunk -> chunk.getContent().contains(".large-component"))); + } + + @Test + void recoversUsefulChunksFromMalformedCss() { + var chunks = extract(".broken { color: red; .nested { display: block; }"); + + assertFalse(chunks.isEmpty()); + assertTrue(chunks.stream().anyMatch(chunk -> chunk.getContent().contains("color"))); + } + + private java.util.List extract(String content) { + var codebase = Codebase.builder().id(UUID.randomUUID()).name("test").build(); + var file = RepositoryFile.builder() + .id(UUID.randomUUID()) + .codebase(codebase) + .path("styles.css") + .language("css") + .build(); + ParsedFile parsed = parser.parse(file, content); + return extractor.extract(parsed); + } +} diff --git a/server/src/test/java/com/meet/server/feature/indexing/extractor/HtmlExtractorTest.java b/server/src/test/java/com/meet/server/feature/indexing/extractor/HtmlExtractorTest.java new file mode 100644 index 0000000..78d11fb --- /dev/null +++ b/server/src/test/java/com/meet/server/feature/indexing/extractor/HtmlExtractorTest.java @@ -0,0 +1,70 @@ +package com.meet.server.feature.indexing.extractor; + +import com.meet.server.feature.codebase.Codebase; +import com.meet.server.feature.indexing.parser.ParsedFile; +import com.meet.server.feature.indexing.parser.TreeSitterParser; +import com.meet.server.feature.repositoryfile.RepositoryFile; +import org.junit.jupiter.api.Test; + +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.*; + +class HtmlExtractorTest { + + private final TreeSitterParser parser = new TreeSitterParser(); + private final HtmlExtractor extractor = new HtmlExtractor(); + + @Test + void emitsHtmlElementsAsSourceAwareChunks() { + var chunks = extract(""" + + + +

Hello

World

+ + + """); + + assertFalse(chunks.isEmpty()); + assertTrue(chunks.stream().anyMatch(chunk -> chunk.getContent().contains(" chunk.getContent().contains("

Hello

"))); + assertTrue(chunks.stream().allMatch(chunk -> chunk.getStartLine() != null && chunk.getEndLine() != null)); + } + + @Test + void splitsLargeElementWithoutLosingOpeningTagContext() { + var source = new StringBuilder("
\n"); + for (int i = 0; i < 300; i++) { + source.append("

Result ").append(i).append("

") + .append("x".repeat(20)).append("

\n"); + } + source.append("
\n"); + + var chunks = extract(source.toString()); + + assertTrue(chunks.size() > 1); + assertTrue(chunks.stream().allMatch(chunk -> chunk.getContent().length() <= 4_000)); + assertTrue(chunks.stream().anyMatch(chunk -> chunk.getContent().contains("section class=\"results\""))); + } + + @Test + void recoversUsefulChunksFromMalformedTemplateMarkup() { + var chunks = extract("
Visible"); + + assertFalse(chunks.isEmpty()); + assertTrue(chunks.stream().anyMatch(chunk -> chunk.getContent().contains("Visible"))); + } + + private java.util.List extract(String content) { + var codebase = Codebase.builder().id(UUID.randomUUID()).name("test").build(); + var file = RepositoryFile.builder() + .id(UUID.randomUUID()) + .codebase(codebase) + .path("index.html") + .language("html") + .build(); + ParsedFile parsed = parser.parse(file, content); + return extractor.extract(parsed); + } +} diff --git a/server/src/test/java/com/meet/server/feature/indexing/extractor/JsonExtractorTest.java b/server/src/test/java/com/meet/server/feature/indexing/extractor/JsonExtractorTest.java new file mode 100644 index 0000000..e321916 --- /dev/null +++ b/server/src/test/java/com/meet/server/feature/indexing/extractor/JsonExtractorTest.java @@ -0,0 +1,66 @@ +package com.meet.server.feature.indexing.extractor; + +import com.meet.server.feature.codebase.Codebase; +import com.meet.server.feature.indexing.parser.JsonParser; +import com.meet.server.feature.repositoryfile.RepositoryFile; +import org.junit.jupiter.api.Test; +import tools.jackson.databind.json.JsonMapper; + +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.*; + +class JsonExtractorTest { + + private final JsonMapper jsonMapper = JsonMapper.builder().build(); + private final JsonParser parser = new JsonParser(jsonMapper); + private final JsonExtractor extractor = new JsonExtractor(jsonMapper); + + @Test + void emitsPathAwareChunksForNestedJson() { + var chunks = extract("{\"name\":\"demo\",\"dependencies\":{\"spring\":\"4.1\"}}"); + + assertEquals(1, chunks.size()); + assertTrue(chunks.getFirst().getContent().contains("JSON path: $")); + assertTrue(chunks.getFirst().getContent().contains("dependencies")); + } + + @Test + void splitsLargeObjectsIntoNestedPathChunks() { + var source = new StringBuilder("{\"dependencies\":{"); + for (int i = 0; i < 300; i++) { + if (i > 0) source.append(','); + source.append("\"dependency").append(i).append("\":\"").append("x".repeat(20)).append("\""); + } + source.append("}}"); + + var chunks = extract(source.toString()); + + assertTrue(chunks.size() > 1); + assertTrue(chunks.stream().allMatch(chunk -> chunk.getContent().length() <= 4_000)); + assertTrue(chunks.stream().anyMatch(chunk -> chunk.getContent().contains("$.dependencies."))); + assertEquals(0, chunks.getFirst().getChunkIndex()); + } + + @Test + void malformedJsonFallsBackToTextLanguage() { + var parsed = parser.parse(file(), "{ malformed"); + + assertEquals(com.meet.server.feature.indexing.language.Language.UNKNOWN, parsed.language()); + assertNull(parsed.jsonTree()); + } + + private java.util.List extract(String content) { + return extractor.extract(parser.parse(file(), content)); + } + + private RepositoryFile file() { + var codebase = Codebase.builder().id(UUID.randomUUID()).name("test").build(); + return RepositoryFile.builder() + .id(UUID.randomUUID()) + .codebase(codebase) + .path("package.json") + .language("json") + .build(); + } +} diff --git a/server/src/test/java/com/meet/server/feature/indexing/extractor/TreeSitterExtractorTest.java b/server/src/test/java/com/meet/server/feature/indexing/extractor/TreeSitterExtractorTest.java new file mode 100644 index 0000000..541019c --- /dev/null +++ b/server/src/test/java/com/meet/server/feature/indexing/extractor/TreeSitterExtractorTest.java @@ -0,0 +1,116 @@ +package com.meet.server.feature.indexing.extractor; + +import com.meet.server.feature.codebase.Codebase; +import com.meet.server.feature.indexing.parser.ParsedFile; +import com.meet.server.feature.indexing.parser.TreeSitterParser; +import com.meet.server.feature.repositoryfile.RepositoryFile; +import org.junit.jupiter.api.Test; + +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.*; + +class TreeSitterExtractorTest { + + private final TreeSitterParser parser = new TreeSitterParser(); + private final TreeSitterExtractor extractor = new TreeSitterExtractor(); + + @Test + void ignoresPackageAndImports() { + var chunks = extract(""" + package com.example; + import java.util.List; + + public class Demo { + public List names() { + return List.of("demo"); + } + } + """); + + assertFalse(chunks.isEmpty()); + assertTrue(chunks.stream().noneMatch(chunk -> chunk.getContent().contains("import java.util.List"))); + assertTrue(chunks.stream().noneMatch(chunk -> chunk.getContent().contains("package com.example"))); + } + + @Test + void splitsLargeClassIntoBoundedMemberChunksWithContext() { + var source = new StringBuilder("public class LargeService {\n"); + for (int i = 0; i < 40; i++) { + source.append(" public String method").append(i).append("() {\n") + .append(" return \"").append("x".repeat(130)).append("\";\n") + .append(" }\n"); + } + source.append("}\n"); + + var chunks = extract(source.toString()); + + assertTrue(chunks.size() > 1); + assertTrue(chunks.stream().allMatch(chunk -> chunk.getContent().length() <= 4_000)); + assertTrue(chunks.stream().allMatch(chunk -> chunk.getContent().contains("LargeService"))); + assertEquals(0, chunks.getFirst().getChunkIndex()); + assertEquals(1, chunks.get(1).getChunkIndex()); + } + + @Test + void preservesTsxAsCompleteComponentSource() { + var chunks = extract("Demo.tsx", """ + import { useState } from "react"; + + type Props = { title: string }; + + export function Demo({ title }: Props) { + const [open, setOpen] = useState(false); + return ( +
+

{title}

+ +
+ ); + } + """); + + assertFalse(chunks.isEmpty()); + assertTrue(chunks.stream().anyMatch(chunk -> chunk.getContent().contains("
") + && chunk.getContent().contains("

{title}

") + && chunk.getContent().contains("
"))); + assertTrue(chunks.stream().noneMatch(chunk -> chunk.getContent().equals("Demo") + || chunk.getContent().equals("()") + || chunk.getContent().equals("className"))); + } + + @Test + void preservesJsxAsCompleteComponentSource() { + var chunks = extract("Demo.jsx", """ + export default function Demo() { + return ( +
+
+

Hello {name}

+
+ ); + } + """); + + assertFalse(chunks.isEmpty()); + assertTrue(chunks.stream().anyMatch(chunk -> chunk.getContent().contains("
") + && chunk.getContent().contains("

Hello {name}

") + && chunk.getContent().contains(""))); + } + + private java.util.List extract(String content) { + return extract("Demo.java", content); + } + + private java.util.List extract(String path, String content) { + var codebase = Codebase.builder().id(UUID.randomUUID()).name("test").build(); + var file = RepositoryFile.builder() + .id(UUID.randomUUID()) + .codebase(codebase) + .path(path) + .language(path.substring(path.lastIndexOf('.') + 1)) + .build(); + ParsedFile parsed = parser.parse(file, content); + return extractor.extract(parsed); + } +} diff --git a/server/src/test/java/com/meet/server/feature/indexing/language/LanguageTest.java b/server/src/test/java/com/meet/server/feature/indexing/language/LanguageTest.java new file mode 100644 index 0000000..17000ef --- /dev/null +++ b/server/src/test/java/com/meet/server/feature/indexing/language/LanguageTest.java @@ -0,0 +1,34 @@ +package com.meet.server.feature.indexing.language; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class LanguageTest { + + @Test + void mapsProgrammingAliasesToGrammarBackedLanguages() { + for (var language : new Language[]{ + Language.JAVA, Language.KOTLIN, Language.PYTHON, Language.JAVASCRIPT, + Language.TYPESCRIPT, Language.TSX, Language.GO, Language.RUST, Language.C, Language.CPP, + Language.CSHARP, Language.PHP, Language.RUBY, Language.SWIFT + }) { + assertTrue(language.isProgramming()); + assertTrue(language.hasGrammar()); + } + + assertEquals(Language.TSX, Language.from("tsx")); + assertEquals(Language.JAVASCRIPT, Language.from("jsx")); + assertEquals(Language.CSHARP, Language.from(".cs")); + assertEquals("java", Language.extensionOf("src/main/Demo.java")); + assertEquals(Language.ParserKind.JSON, Language.JSON.parserKind()); + assertEquals(Language.ParserKind.MARKDOWN, Language.MARKDOWN.parserKind()); + assertEquals(Language.HTML, Language.from("html")); + assertEquals(Language.HTML, Language.from(".htm")); + assertEquals(Language.CSS, Language.from("css")); + assertTrue(Language.HTML.hasGrammar()); + assertTrue(Language.CSS.hasGrammar()); + assertEquals(Language.ParserKind.TEXT, Language.UNKNOWN.parserKind()); + } +}