From ceb2b0cf53a3d9f94c3285b898d630741398312f Mon Sep 17 00:00:00 2001 From: Arthur Chan Date: Wed, 3 Jul 2024 16:40:46 +0000 Subject: [PATCH 1/2] JVM: Revamp prompt structure Signed-off-by: Arthur Chan --- llm_toolkit/prompt_builder.py | 2 +- prompts/template_xml/jvm_base.txt | 2 +- prompts/template_xml/jvm_problem.txt | 8 ++++++++ prompts/template_xml/jvm_problem_constructor.txt | 11 ----------- prompts/template_xml/jvm_problem_method.txt | 11 ----------- 5 files changed, 10 insertions(+), 24 deletions(-) diff --git a/llm_toolkit/prompt_builder.py b/llm_toolkit/prompt_builder.py index 1db409ac4c..9508ab986b 100644 --- a/llm_toolkit/prompt_builder.py +++ b/llm_toolkit/prompt_builder.py @@ -714,7 +714,7 @@ def _format_arguments(self) -> str: argument_descriptions.append(argument) - return '\n'.join(argument_descriptions) + return '' + '\n'.join(argument_descriptions) + '' def _format_source_reference(self, signature: str) -> Tuple[str, str]: """Formats the source code reference for this target.""" diff --git a/prompts/template_xml/jvm_base.txt b/prompts/template_xml/jvm_base.txt index b349197a99..479b6e79ac 100644 --- a/prompts/template_xml/jvm_base.txt +++ b/prompts/template_xml/jvm_base.txt @@ -4,4 +4,4 @@ The tag contains information of the target method to invoke. The tag contains information of each of the target method arguments. The tag contains a list of exceptions thrown by the target method that you MUST catch. The tag contains constructor or method call details you MUST use to create the needed object before calling the target method. -The tag contains additional requirements that you MUST follow for this code generation. +The tag contains additional requirements that you MUST follow for this code generation. diff --git a/prompts/template_xml/jvm_problem.txt b/prompts/template_xml/jvm_problem.txt index 7adebf37ec..1d4248daba 100644 --- a/prompts/template_xml/jvm_problem.txt +++ b/prompts/template_xml/jvm_problem.txt @@ -1,6 +1,14 @@ Your goal is to write a fuzzing harness for the provided method signature using Jazzer framework from Code Intellengence. It is important that the provided solution compiles and actually calls the function specified by the method signature: {TARGET} +{ARGUMENTS} +{EXCEPTIONS} +Here is the source code of the target constructor for reference. + +{SELF_SOURCE} + +Here is a list of source codes of methods that directly invoke the target consturctor for reference. +{CROSS_SOURCE} {REQUIREMENTS} {DATA_MAPPING} diff --git a/prompts/template_xml/jvm_problem_constructor.txt b/prompts/template_xml/jvm_problem_constructor.txt index 93ecfc9a13..3c2f55c4b0 100644 --- a/prompts/template_xml/jvm_problem_constructor.txt +++ b/prompts/template_xml/jvm_problem_constructor.txt @@ -7,15 +7,4 @@ The constructor signature follows the format of [Full qualified name of th For example, for the constructor of class Test of package org.test which takes in a single integer would have the following signature: [org.test.Test].(int) The target method is belonging to the Java project {PROJECT_NAME} ({PROJECT_URL}). -Here is the list of arguments of the target constructor with descriptions. - -{ARGUMENTS} - -{EXCEPTIONS} -Here is the source code of the target constructor for reference. - -{SELF_SOURCE} - -Here is a list of source codes of methods that directly invoke the target consturctor for reference. -{CROSS_SOURCE} diff --git a/prompts/template_xml/jvm_problem_method.txt b/prompts/template_xml/jvm_problem_method.txt index 5cde6c145e..2a5128c168 100644 --- a/prompts/template_xml/jvm_problem_method.txt +++ b/prompts/template_xml/jvm_problem_method.txt @@ -6,15 +6,4 @@ The method signature follows the format of [Full qualified name of the cla For example, for a method test in class Test of package org.test which takes in a single integer would have the following method signature: [org.test.Test].test(int) The target method is belonging to the Java project {PROJECT_NAME} ({PROJECT_URL}). -Here is the list of arguments of the target method with descriptions. - -{ARGUMENTS} - -{EXCEPTIONS} -Here is the source code of the target method for reference. - -{SELF_SOURCE} - -Here is a list of source codes of methods that directly invoke the target method for reference. -{CROSS_SOURCE} From e1835939f602352901ce03b414ad83f138759265 Mon Sep 17 00:00:00 2001 From: Arthur Chan Date: Wed, 3 Jul 2024 17:23:14 +0000 Subject: [PATCH 2/2] Fix logic Signed-off-by: Arthur Chan --- llm_toolkit/prompt_builder.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/llm_toolkit/prompt_builder.py b/llm_toolkit/prompt_builder.py index 9508ab986b..2b3ffe8efb 100644 --- a/llm_toolkit/prompt_builder.py +++ b/llm_toolkit/prompt_builder.py @@ -662,11 +662,9 @@ def _format_target(self, signature: str) -> str: method and format it for the prompts creation. """ if '' in signature: - target = self._format_target_constructor(signature) - else: - target = self._format_target_method(signature) + return self._format_target_constructor(signature) - return target.replace('{EXCEPTIONS}', self._format_exceptions()) + return self._format_target_method(signature) def _format_requirement(self, signature: str) -> str: """Formats a requirement based on the prompt template.""" @@ -742,6 +740,7 @@ def _format_problem(self, signature: str) -> str: self._format_requirement(signature)) problem = problem.replace('{DATA_MAPPING}', self._format_data_filler()) problem = problem.replace('{ARGUMENTS}', self._format_arguments()) + problem = problem.replace('{EXCEPTIONS}', self._format_exceptions()) self_source, cross_source = self._format_source_reference(signature) problem = problem.replace('{SELF_SOURCE}', self_source)