Skip to content

Commit

Permalink
Add argument randomization (#368)
Browse files Browse the repository at this point in the history
  • Loading branch information
@bernhl
bernhl authored Sep 21, 2022
1 parent d754a13 commit 608e4c4
Show file tree
Hide file tree
Showing 14 changed files with 136 additions and 36 deletions.
2 changes: 1 addition & 1 deletion Sources/Fuzzilli/Execution/REPRL.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ public class REPRL: ComponentBase, ScriptRunner {
private let maxExecsBeforeRespawn = 1000

/// Commandline arguments for the executable
private let processArguments: [String]
public private(set) var processArguments: [String]

/// Environment variables for the child process
private var env = [String]()
Expand Down
2 changes: 2 additions & 0 deletions Sources/Fuzzilli/Execution/ScriptRunner.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
// limitations under the License.

public protocol ScriptRunner: Component {
var processArguments: [String] { get }

/// Executes a script, waits for it to complete, and returns the result.
func run(_ script: String, withTimeout timeout: UInt32) -> Execution

Expand Down
1 change: 1 addition & 0 deletions Sources/Fuzzilli/Fuzzer.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -646,6 +646,7 @@ public class Fuzzer {
program.comments.add("TERMSIG: \(termsig)\n", at: .footer)
program.comments.add("STDERR:\n" + stderr, at: .footer)
program.comments.add("STDOUT:\n" + stdout, at: .footer)
program.comments.add("ARGS: \(runner.processArguments.joined(separator: " "))\n", at: .footer)
}
Assert(program.comments.at(.footer)?.contains("CRASH INFO") ?? false)

Expand Down
2 changes: 2 additions & 0 deletions Sources/Fuzzilli/Util/MockFuzzer.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ struct MockExecution: Execution {
}

class MockScriptRunner: ScriptRunner {
var processArguments: [String] = []

func run(_ script: String, withTimeout timeout: UInt32) -> Execution {
return MockExecution(outcome: .succeeded,
stdout: "",
Expand Down
4 changes: 3 additions & 1 deletion Sources/FuzzilliCli/Profiles/DuktapeProfile.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@
import Fuzzilli

let duktapeProfile = Profile(
processArguments: ["--reprl"],
getProcessArguments: { (randomizingArguments: Bool) -> [String] in
return ["--reprl"]
},

processEnv: ["UBSAN_OPTIONS": "handle_segv=0"],

Expand Down
43 changes: 31 additions & 12 deletions Sources/FuzzilliCli/Profiles/JSCProfile.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,18 +31,37 @@ fileprivate let ForceFTLCompilationGenerator = CodeGenerator("ForceFTLCompilatio
}

let jscProfile = Profile(
processArguments: ["--validateOptions=true",
// No need to call functions thousands of times before they are JIT compiled
"--thresholdForJITSoon=10",
"--thresholdForJITAfterWarmUp=10",
"--thresholdForOptimizeAfterWarmUp=100",
"--thresholdForOptimizeAfterLongWarmUp=100",
"--thresholdForOptimizeSoon=100",
"--thresholdForFTLOptimizeAfterWarmUp=1000",
"--thresholdForFTLOptimizeSoon=1000",
// Enable bounds check elimination validation
"--validateBCE=true",
"--reprl"],
getProcessArguments: { (randomizingArguments: Bool) -> [String] in
var args = [
"--validateOptions=true",
// No need to call functions thousands of times before they are JIT compiled
"--thresholdForJITSoon=10",
"--thresholdForJITAfterWarmUp=10",
"--thresholdForOptimizeAfterWarmUp=100",
"--thresholdForOptimizeAfterLongWarmUp=100",
"--thresholdForOptimizeSoon=100",
"--thresholdForFTLOptimizeAfterWarmUp=1000",
"--thresholdForFTLOptimizeSoon=1000",
// Enable bounds check elimination validation
"--validateBCE=true",
"--reprl"]

guard randomizingArguments else { return args }

args.append("--useBaselineJIT=\(probability(0.9) ? "true" : "false")")
args.append("--useDFGJIT=\(probability(0.9) ? "true" : "false")")
args.append("--useFTLJIT=\(probability(0.9) ? "true" : "false")")
args.append("--useRegExpJIT=\(probability(0.9) ? "true" : "false")")
args.append("--useTailCalls=\(probability(0.9) ? "true" : "false")")
args.append("--optimizeRecursiveTailCalls=\(probability(0.9) ? "true" : "false")")
args.append("--useObjectAllocationSinking=\(probability(0.9) ? "true" : "false")")
args.append("--useArityFixupInlining=\(probability(0.9) ? "true" : "false")")
args.append("--useValueRepElimination=\(probability(0.9) ? "true" : "false")")
args.append("--useArchitectureSpecificOptimizations=\(probability(0.9) ? "true" : "false")")
args.append("--useAccessInlining=\(probability(0.9) ? "true" : "false")")

return args
},

processEnv: ["UBSAN_OPTIONS":"handle_segv=0"],

Expand Down
4 changes: 3 additions & 1 deletion Sources/FuzzilliCli/Profiles/JerryscriptProfile.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@
import Fuzzilli

let jerryscriptProfile = Profile(
processArguments: ["--reprl-fuzzilli"],
getProcessArguments: { (randomizingArguments: Bool) -> [String] in
return ["--reprl-fuzzilli"]
},

// processEnv: [:],
processEnv: ["UBSAN_OPTIONS":"handle_segv=0"],
Expand Down
2 changes: 1 addition & 1 deletion Sources/FuzzilliCli/Profiles/Profile.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
import Fuzzilli

struct Profile {
let processArguments: [String]
let getProcessArguments: (_: Bool) -> [String]
let processEnv: [String : String]
let codePrefix: String
let codeSuffix: String
Expand Down
4 changes: 3 additions & 1 deletion Sources/FuzzilliCli/Profiles/QjsProfile.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@
import Fuzzilli

let qjsProfile = Profile(
processArguments: ["--reprl"],
getProcessArguments: { (randomizingArguments: Bool) -> [String] in
return ["--reprl"]
},

processEnv: ["UBSAN_OPTIONS": "handle_segv=0"],

Expand Down
5 changes: 4 additions & 1 deletion Sources/FuzzilliCli/Profiles/QtjsProfile.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,10 @@ fileprivate let ForceQV4JITGenerator = CodeGenerator("ForceQV4JITGenerator", inp
}

let qtjsProfile = Profile(
processArguments: ["-reprl"],
getProcessArguments: { (randomizingArguments: Bool) -> [String] in
return ["-reprl"]
},

processEnv: ["UBSAN_OPTIONS":"handle_segv=0"],

codePrefix: """
Expand Down
48 changes: 40 additions & 8 deletions Sources/FuzzilliCli/Profiles/SpidermonkeyProfile.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,46 @@ fileprivate let ForceSpidermonkeyIonGenerator = CodeGenerator("ForceSpidermonkey
}

let spidermonkeyProfile = Profile(
processArguments: [
"--baseline-warmup-threshold=10",
"--ion-warmup-threshold=100",
"--ion-check-range-analysis",
"--ion-extra-checks",
"--fuzzing-safe",
"--reprl",
],
getProcessArguments: { (randomizingArguments: Bool) -> [String] in
var args = [
"--baseline-warmup-threshold=10",
"--ion-warmup-threshold=100",
"--ion-check-range-analysis",
"--ion-extra-checks",
"--fuzzing-safe",
"--reprl"]

guard randomizingArguments else { return args }

args.append("--small-function-length=\(1<<Int.random(in: 7...12))")
args.append("--inlining-entry-threshold=\(1<<Int.random(in: 2...10))")
args.append("--gc-zeal=\(probability(0.5) ? UInt32(0) : UInt32(Int.random(in: 1...24)))")
args.append("--ion-scalar-replacement=\(probability(0.9) ? "on": "off")")
args.append("--ion-pruning=\(probability(0.9) ? "on": "off")")
args.append("--ion-range-analysis=\(probability(0.9) ? "on": "off")")
args.append("--ion-inlining=\(probability(0.9) ? "on": "off")")
args.append("--ion-gvn=\(probability(0.9) ? "on": "off")")
args.append("--ion-osr=\(probability(0.9) ? "on": "off")")
args.append("--ion-edgecase-analysis=\(probability(0.9) ? "on": "off")")
args.append("--nursery-size=\(1<<Int.random(in: 0...5))")
args.append("--nursery-strings=\(probability(0.9) ? "on": "off")")
args.append("--nursery-bigints=\(probability(0.9) ? "on": "off")")
args.append("--spectre-mitigations=\(probability(0.1) ? "on": "off")")
if probability(0.1) {
args.append("--no-native-regexp")
}
args.append("--ion-optimize-shapeguards=\(probability(0.9) ? "on": "off")")
args.append("--ion-licm=\(probability(0.9) ? "on": "off")")
args.append("--ion-instruction-reordering=\(probability(0.9) ? "on": "off")")
args.append("--cache-ir-stubs=\(probability(0.9) ? "on": "off")")
args.append(chooseUniform(from: ["--no-sse3", "--no-ssse3", "--no-sse41", "--no-sse42", "--enable-avx"]))
if probability(0.1) {
args.append("--ion-regalloc=testbed")
}
args.append(probability(0.9) ? "--enable-watchtower" : "--disable-watchtower")
args.append("--ion-sink=\(probability(0.0) ? "on": "off")") // disabled
return args
},

processEnv: ["UBSAN_OPTIONS": "handle_segv=0"],

Expand Down
47 changes: 39 additions & 8 deletions Sources/FuzzilliCli/Profiles/V8Profile.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,14 +231,45 @@ fileprivate let VerifyTypeTemplate = ProgramTemplate("VerifyTypeTemplate") { b i
}

let v8Profile = Profile(
processArguments: ["--expose-gc",
"--future",
"--harmony",
"--assert-types",
"--harmony-rab-gsab",
"--allow-natives-syntax",
"--interrupt-budget=1024",
"--fuzzing"],
getProcessArguments: { (randomizingArguments: Bool) -> [String] in
var args = [
"--expose-gc",
"--future",
"--harmony",
"--assert-types",
"--harmony-rab-gsab",
"--allow-natives-syntax",
"--interrupt-budget=1024",
"--fuzzing"]

guard randomizingArguments else { return args }

args.append(probability(0.9) ? "--sparkplug" : "--no-sparkplug")
args.append(probability(0.9) ? "--opt" : "--no-opt")
args.append(probability(0.9) ? "--lazy" : "--no-lazy")
args.append(probability(0.1) ? "--always-opt" : "--no-always-opt")
args.append(probability(0.1) ? "--always-osr" : "--no-always-osr")
args.append(probability(0.1) ? "--force-slow-path" : "--no-force-slow-path")
args.append(probability(0.9) ? "--turbo-move-optimization" : "--no-turbo-move-optimization")
args.append(probability(0.9) ? "--turbo-jt" : "--no-turbo-jt")
args.append(probability(0.9) ? "--turbo-loop-peeling" : "--no-turbo-loop-peeling")
args.append(probability(0.9) ? "--turbo-loop-variable" : "--no-turbo-loop-variable")
args.append(probability(0.9) ? "--turbo-loop-rotation" : "--no-turbo-loop-rotation")
args.append(probability(0.9) ? "--turbo-cf-optimization" : "--no-turbo-cf-optimization")
args.append(probability(0.9) ? "--turbo-escape" : "--no-turbo-escape")
args.append(probability(0.9) ? "--turbo-allocation-folding" : "--no-turbo-allocation-folding")
args.append(probability(0.9) ? "--turbo-instruction-scheduling" : "--no-turbo-instruction-scheduling")
args.append(probability(0.9) ? "--turbo-stress-instruction-scheduling" : "--no-turbo-stress-instruction-scheduling")
args.append(probability(0.9) ? "--turbo-store-elimination" : "--no-turbo-store-elimination")
args.append(probability(0.9) ? "--turbo-rewrite-far-jumps" : "--no-turbo-rewrite-far-jumps")
args.append(probability(0.9) ? "--turbo-optimize-apply" : "--no-turbo-optimize-apply")
args.append(chooseUniform(from: ["--no-enable-sse3", "--no-enable-ssse3", "--no-enable-sse4-1", "--no-enable-sse4-2", "--no-enable-avx", "--no-enable-avx2",]))
args.append(probability(0.9) ? "--turbo-load-elimination" : "--no-turbo-load-elimination")
args.append(probability(0.9) ? "--turbo-inlining" : "--no-turbo-inlining")
args.append(probability(0.9) ? "--turbo-splitting" : "--no-turbo-splitting")

return args
},

processEnv: [:],

Expand Down
4 changes: 3 additions & 1 deletion Sources/FuzzilliCli/Profiles/XSProfile.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@
import Fuzzilli

let xsProfile = Profile(
processArguments: ["-f"],
getProcessArguments: { (randomizingArguments: Bool) -> [String] in
return ["-f"]
},

processEnv: ["UBSAN_OPTIONS":"handle_segv=0"],

Expand Down
4 changes: 3 additions & 1 deletion Sources/FuzzilliCli/main.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ Options:
types: Programs written to disk also contain variable type information as
determined by Fuzzilli as comments
all: All of the above
--argumentRandomization : Enable JS engine argument randomization
""")
exit(0)
}
Expand Down Expand Up @@ -163,6 +164,7 @@ let collectRuntimeTypes = args.has("--collectRuntimeTypes")
let diagnostics = args.has("--diagnostics")
let inspect = args["--inspect"]
let swarmTesting = args.has("--swarmTesting")
let randomizingArguments = args.has("--argumentRandomization")

guard numJobs >= 1 else {
configError("Must have at least 1 job")
Expand Down Expand Up @@ -365,7 +367,7 @@ for (generator, var weight) in (additionalCodeGenerators + regularCodeGenerators

func makeFuzzer(for profile: Profile, with configuration: Configuration) -> Fuzzer {
// A script runner to execute JavaScript code in an instrumented JS engine.
let runner = REPRL(executable: jsShellPath, processArguments: profile.processArguments, processEnvironment: profile.processEnv)
let runner = REPRL(executable: jsShellPath, processArguments: profile.getProcessArguments(randomizingArguments), processEnvironment: profile.processEnv)

let engine: FuzzEngine
switch engineName {
Expand Down

3 comments on commit 608e4c4

@turnerrocks1
Copy link
Contributor

@turnerrocks1 turnerrocks1 commented on 608e4c4 Sep 22, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just a feature idea or specific question, though this is an awesome idea say a crash happens and it chooses random args in this manor one wouldn't know what arg with said poc triggered the crash, also with that in mind I feel as also you should add something to pass to the fuzzer so it could reuse the same args that triggered the crash for example...

lets say poc.js : "let o = {}" triggered a crash with --useBaselineJIT=true,
and this triggered the crash and lets say this would've stayed deterministic if the flag stayed the same.
But on the other hand lets say the randomization flips it to --useBaselineJIT=false,
and the crash doesn't happen wouldn't it turn the case a flaky ?

Considering the fact that when determining determinism it executes it I believe a default of 3 times reference :

fuzzilli/Sources/FuzzilliCli/main.swift

Line 139 in 608e4c4

let minDeterminismExecs = args.int(for: "--minDeterminismExecs") ?? 3

IDK how @saelo fuzzer goes after that and what file actually does the execution probably would be Fuzzer.swift but this would definitely be an issue if crashes are found by a user.

At least if your not going to add the above the minimum is to output args to crash output that be at the bottom of the crashing js file like such :

"// CRASH INFO
// ==========
//Args : --useBaselineJIT=true --useConcurrentJIT=false etc etc.
// TERMSIG: 11
// STDERR:
// ASSERTION FAILED: [redacted]
// /Users/bootywarrior/Downloads/Webkit/latest/WebKit/Source/JavaScriptCore/runtime/[redacted]
// 1 0x102efd714 WTFCrash
[redacted]
// 16 0x100e533e8 llint_entry
// 17 0x100e2db04 vmEntryToJavaScript
// 18 0x102111eac JSC::Interpreter::executeProgram(JSC::SourceCode const&, JSC::JSGlobalObject*, JSC::JSObject*)
// 19 0x10255b210 JSC::evaluate(JSC::JSGlobalObject*, JSC::SourceCode const&, JSC::JSValue, WTF::NakedPtrJSC::Exception&)
// 20 0x100dd63a0 jscmain(int, char**)
// 21 0x100dd4d48 main
// 22 0x104e9908c
"

@saelo
Copy link
Collaborator

@saelo saelo commented on 608e4c4 Sep 23, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The PR already adds the process arguments to the "CRASH INFO" comment so they are available for reproducing crashes locally here: 608e4c4#diff-e524fe9b24bf74c95c52ee3d975a12c5352c04d22436945653f33828e38ada06R649
I think this is what you mean? Crashes may still be marked as flaky if they don't reproduce on other instances, but that's exactly what flaky means...

@turnerrocks1
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@saelo ahh I see so that is actually handled by MockFuzzer.swift I was looking in other places and thought It was missing :) that's great!

Please sign in to comment.