fix: replace recursive analysis algorithms with iterations to avoid...

fix: replace recursive analysis algorithms with iterations to avoid StackOverflow on big methods (#441)

jadx-core/src/main/java/jadx/core/dex/visitors/blocksmaker/BlockProcessor.java

@@ -3,7 +3,9 @@ package jadx.core.dex.visitors.blocksmaker;
 import java.util.ArrayList;
 import java.util.BitSet;
 import java.util.Collections;
+import java.util.Deque;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 
 import org.slf4j.Logger;
@@ -25,7 +27,6 @@ import jadx.core.dex.trycatch.ExceptionHandler;
 import jadx.core.dex.trycatch.TryCatchBlock;
 import jadx.core.dex.visitors.AbstractVisitor;
 import jadx.core.utils.BlockUtils;
-import jadx.core.utils.exceptions.JadxOverflowException;
 import jadx.core.utils.exceptions.JadxRuntimeException;
 
 import static jadx.core.dex.visitors.blocksmaker.BlockSplitter.connect;
@@ -220,7 +221,12 @@ public class BlockProcessor extends AbstractVisitor {
 		markLoops(mth);
 
 		// clear self dominance
-		basicBlocks.forEach(block -> block.getDoms().clear(block.getId()));
+		basicBlocks.forEach(block -> {
+			block.getDoms().clear(block.getId());
+			if (block.getDoms().isEmpty()) {
+				block.setDoms(EMPTY);
+			}
+		});
 
 		// calculate immediate dominators
 		for (BlockNode block : basicBlocks) {
@@ -253,11 +259,20 @@ public class BlockProcessor extends AbstractVisitor {
 		for (BlockNode exit : mth.getExitBlocks()) {
 			exit.setDomFrontier(EMPTY);
 		}
-		for (BlockNode block : mth.getBasicBlocks()) {
+		List<BlockNode> domSortedBlocks = new ArrayList<>(mth.getBasicBlocks().size());
+		Deque<BlockNode> stack = new LinkedList<>();
+		stack.push(mth.getEnterBlock());
+		while (!stack.isEmpty()) {
+			BlockNode node = stack.pop();
+			for (BlockNode dominated : node.getDominatesOn()) {
+				stack.push(dominated);
+			}
+			domSortedBlocks.add(node);
+		}
+		Collections.reverse(domSortedBlocks);
+		for (BlockNode block : domSortedBlocks) {
 			try {
 				computeBlockDF(mth, block);
-			} catch (StackOverflowError e) {
-				throw new JadxOverflowException("Failed compute block dominance frontier");
 			} catch (Exception e) {
 				throw new JadxRuntimeException("Failed compute block dominance frontier", e);
 			}
@@ -268,7 +283,6 @@ public class BlockProcessor extends AbstractVisitor {
 		if (block.getDomFrontier() != null) {
 			return;
 		}
-		block.getDominatesOn().forEach(domBlock -> computeBlockDF(mth, domBlock));
 		List<BlockNode> blocks = mth.getBasicBlocks();
 		BitSet domFrontier = null;
 		for (BlockNode s : block.getSuccessors()) {
@@ -281,6 +295,9 @@ public class BlockProcessor extends AbstractVisitor {
 		}
 		for (BlockNode c : block.getDominatesOn()) {
 			BitSet frontier = c.getDomFrontier();
+			if (frontier == null) {
+				throw new JadxRuntimeException("Dominance frontier not calculated for dominated block: " + c + ", from: " + block);
+			}
 			for (int p = frontier.nextSetBit(0); p >= 0; p = frontier.nextSetBit(p + 1)) {
 				if (blocks.get(p).getIDom() != block) {
 					if (domFrontier == null) {
@@ -290,7 +307,7 @@ public class BlockProcessor extends AbstractVisitor {
 				}
 			}
 		}
-		if (domFrontier == null || domFrontier.cardinality() == 0) {
+		if (domFrontier == null || domFrontier.isEmpty()) {
 			domFrontier = EMPTY;
 		}
 		block.setDomFrontier(domFrontier);

jadx-core/src/main/java/jadx/core/dex/visitors/ssa/LiveVarAnalysis.java

@@ -80,35 +80,34 @@ public class LiveVarAnalysis {
 	private void processLiveInfo() {
 		int bbCount = mth.getBasicBlocks().size();
 		int regsCount = mth.getRegsCount();
-		BitSet[] liveIn = initBitSetArray(bbCount, regsCount);
+		BitSet[] liveInBlocks = initBitSetArray(bbCount, regsCount);
 		List<BlockNode> blocks = mth.getBasicBlocks();
-		int blocksSize = blocks.size();
+		int blocksCount = blocks.size();
+		int iterationsLimit = blocksCount * 10;
 		boolean changed;
 		int k = 0;
 		do {
 			changed = false;
-			for (int i = 0; i < blocksSize; i++) {
-				BlockNode block = blocks.get(i);
+			for (BlockNode block : blocks) {
 				int blockId = block.getId();
-				BitSet prevIn = liveIn[blockId];
+				BitSet prevIn = liveInBlocks[blockId];
 				BitSet newIn = new BitSet(regsCount);
-				List<BlockNode> successors = block.getSuccessors();
-				for (int s = 0, successorsSize = successors.size(); s < successorsSize; s++) {
-					newIn.or(liveIn[successors.get(s).getId()]);
+				for (BlockNode successor : block.getSuccessors()) {
+					newIn.or(liveInBlocks[successor.getId()]);
 				}
 				newIn.andNot(defs[blockId]);
 				newIn.or(uses[blockId]);
 				if (!prevIn.equals(newIn)) {
 					changed = true;
-					liveIn[blockId] = newIn;
+					liveInBlocks[blockId] = newIn;
 				}
 			}
-			if (k++ > 1000) {
-				throw new JadxRuntimeException("Live variable analysis reach iterations limit");
+			if (k++ > iterationsLimit) {
+				throw new JadxRuntimeException("Live variable analysis reach iterations limit, blocks count: " + blocksCount);
 			}
 		} while (changed);
 
-		this.liveIn = liveIn;
+		this.liveIn = liveInBlocks;
 	}
 
 	private static BitSet[] initBitSetArray(int length, int bitsCount) {

jadx-core/src/main/java/jadx/core/dex/visitors/ssa/RenameState.java

+package jadx.core.dex.visitors.ssa;
+
+import java.util.Arrays;
+
+import jadx.core.dex.instructions.args.RegisterArg;
+import jadx.core.dex.instructions.args.SSAVar;
+import jadx.core.dex.nodes.BlockNode;
+import jadx.core.dex.nodes.MethodNode;
+
+final class RenameState {
+	private final MethodNode mth;
+	private final BlockNode block;
+	private final SSAVar[] vars;
+	private final int[] versions;
+
+	public static RenameState init(MethodNode mth) {
+		int regsCount = mth.getRegsCount();
+		RenameState state = new RenameState(
+				mth,
+				mth.getEnterBlock(),
+				new SSAVar[regsCount],
+				new int[regsCount]
+		);
+		for (RegisterArg arg : mth.getArguments(true)) {
+			state.startVar(arg);
+		}
+		return state;
+	}
+
+	public static RenameState copyFrom(RenameState state, BlockNode block) {
+		return new RenameState(
+				state.mth,
+				block,
+				Arrays.copyOf(state.vars, state.vars.length),
+				state.versions
+		);
+	}
+
+	private RenameState(MethodNode mth, BlockNode block, SSAVar[] vars, int[] versions) {
+		this.mth = mth;
+		this.block = block;
+		this.vars = vars;
+		this.versions = versions;
+	}
+
+	public BlockNode getBlock() {
+		return block;
+	}
+
+	public SSAVar getVar(int regNum) {
+		return vars[regNum];
+	}
+
+	public void startVar(RegisterArg regArg) {
+		int regNum = regArg.getRegNum();
+		int version = versions[regNum]++;
+		vars[regNum] = mth.makeNewSVar(regNum, version, regArg);
+	}
+}

jadx-core/src/main/java/jadx/core/dex/visitors/ssa/SSATransform.java

 package jadx.core.dex.visitors.ssa;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.BitSet;
 import java.util.Deque;
 import java.util.Iterator;
@@ -120,35 +119,31 @@ public class SSATransform extends AbstractVisitor {
 		if (!mth.getSVars().isEmpty()) {
 			throw new JadxRuntimeException("SSA rename variables already executed");
 		}
-		int regsCount = mth.getRegsCount();
-		SSAVar[] vars = new SSAVar[regsCount];
-		int[] versions = new int[regsCount];
-		// init method arguments
-		for (RegisterArg arg : mth.getArguments(true)) {
-			int regNum = arg.getRegNum();
-			vars[regNum] = newSSAVar(mth, versions, arg, regNum);
+		RenameState initState = RenameState.init(mth);
+		initPhiInEnterBlock(initState);
+
+		Deque<RenameState> stack = new LinkedList<>();
+		stack.push(initState);
+		while (!stack.isEmpty()) {
+			RenameState state = stack.pop();
+			renameVarsInBlock(state);
+			for (BlockNode dominated : state.getBlock().getDominatesOn()) {
+				stack.push(RenameState.copyFrom(state, dominated));
 			}
-		BlockNode enterBlock = mth.getEnterBlock();
-		initPhiInEnterBlock(vars, enterBlock);
-		renameVar(mth, vars, versions, enterBlock);
 		}
-
-	private static SSAVar newSSAVar(MethodNode mth, int[] versions, RegisterArg arg, int regNum) {
-		int version = versions[regNum]++;
-		return mth.makeNewSVar(regNum, version, arg);
 	}
 
-	private static void initPhiInEnterBlock(SSAVar[] vars, BlockNode enterBlock) {
-		PhiListAttr phiList = enterBlock.get(AType.PHI_LIST);
+	private static void initPhiInEnterBlock(RenameState initState) {
+		PhiListAttr phiList = initState.getBlock().get(AType.PHI_LIST);
 		if (phiList != null) {
 			for (PhiInsn phiInsn : phiList.getList()) {
-				bindPhiArg(vars, enterBlock, phiInsn);
+				bindPhiArg(initState, phiInsn);
 			}
 		}
 	}
 
-	private static void renameVar(MethodNode mth, SSAVar[] vars, int[] vers, BlockNode block) {
-		SSAVar[] inputVars = Arrays.copyOf(vars, vars.length);
+	private static void renameVarsInBlock(RenameState state) {
+		BlockNode block = state.getBlock();
 		for (InsnNode insn : block.getInstructions()) {
 			if (insn.getType() != InsnType.PHI) {
 				for (InsnArg arg : insn.getArguments()) {
@@ -157,18 +152,17 @@ public class SSATransform extends AbstractVisitor {
 					}
 					RegisterArg reg = (RegisterArg) arg;
 					int regNum = reg.getRegNum();
-					SSAVar var = vars[regNum];
+					SSAVar var = state.getVar(regNum);
 					if (var == null) {
 						throw new JadxRuntimeException("Not initialized variable reg: " + regNum
-								+ ", insn: " + insn + ", block:" + block + ", method: " + mth);
+								+ ", insn: " + insn + ", block:" + block);
 					}
 					var.use(reg);
 				}
 			}
 			RegisterArg result = insn.getResult();
 			if (result != null) {
-				int regNum = result.getRegNum();
-				vars[regNum] = newSSAVar(mth, vers, result, regNum);
+				state.startVar(result);
 			}
 		}
 		for (BlockNode s : block.getSuccessors()) {
@@ -177,22 +171,18 @@ public class SSATransform extends AbstractVisitor {
 				continue;
 			}
 			for (PhiInsn phiInsn : phiList.getList()) {
-				bindPhiArg(vars, block, phiInsn);
-			}
+				bindPhiArg(state, phiInsn);
 			}
-		for (BlockNode domOn : block.getDominatesOn()) {
-			renameVar(mth, vars, vers, domOn);
 		}
-		System.arraycopy(inputVars, 0, vars, 0, vars.length);
 	}
 
-	private static void bindPhiArg(SSAVar[] vars, BlockNode block, PhiInsn phiInsn) {
+	private static void bindPhiArg(RenameState state, PhiInsn phiInsn) {
 		int regNum = phiInsn.getResult().getRegNum();
-		SSAVar var = vars[regNum];
+		SSAVar var = state.getVar(regNum);
 		if (var == null) {
 			return;
 		}
-		RegisterArg arg = phiInsn.bindArg(block);
+		RegisterArg arg = phiInsn.bindArg(state.getBlock());
 		var.use(arg);
 		var.setUsedInPhi(phiInsn);
 	}