记一次由 Arthas 引起的 Metaspace OOM 问题

Enhanceerror! exception: 

java.lang.InternalError

error happens when enhancing

class: null, check arthas log: 

/path/to/server-log/arthas.log

java.lang.InternalError:nullat
sun.instrument.InstrumentationImpl
.retransformClasses0(NativeMethod)
at
sun.instrument.InstrumentationImpl
.retransformClasses(InstrumentationImpl.java:144)
at
com.taobao.arthas.core.advisor.Enhancer.enh
ance(Enhancer.java:368)
at
com.taobao.arthas.core.command.mon
itor200.EnhancerCommand.enhance(EnhancerCommand.java:149)
at
com.taobao.arthas.core.comma
nd.monitor200.EnhancerComma
nd.process(EnhancerCommand.java:96)
at
com.taobao.arthas.core.shell.command.impl.AnnotatedCommandImpl.process(AnnotatedCommandImpl.java:82)
at com.taobao.arthas.core.shell.command.impl.AnnotatedCommandImpl.access$100(AnnotatedCommandImpl.java:18)
at com.taobao.arthas.core.shell.command.impl.AnnotatedCommandImpl
$ProcessHandler.handle(AnnotatedCommandImpl.java:111)// 

jcmd<pid> VM.metaspace

// arthas-agent-attach/src/main/java/com/taobao/arthas/agent/attach/ArthasAgent.javapublic void init() throws IllegalStateException {// ...// 通过反射调用 ArthasBootstrap bootstrap = ArthasBootstrap.getInstance(inst); Class<?> bootstrapClass = arthasClassLoader.loadClass(ARTHAS_BOOTSTRAP);Object bootstrap = bootstrapClass.getMethod(GET_INSTANCE, Instrumentation.class, Map.class).invoke(null,instrumentation, configMap);boolean isBind = (Boolean) bootstrapClass.getMethod(IS_BIND).invoke(bootstrap);if (!isBind) {StringerrorMsg = "Arthas server port binding failed! Please check $HOME/logs/arthas/arthas.log for more details.";thrownew RuntimeException(errorMsg); }// ...}

private ArthasBootstrap(Instrumentation instrumentation, Map<String, String> args) throws Throwable {// ... shutdown = new Thread("as-shutdown-hooker") { @Overridepublic void run(){ArthasBootstrap.this.destroy(); } };// 

transformerManager = new TransformerManager(instrumentation);Runtime.getRuntime().addShutdownHook(shutdown); }

public TransformerManager(Instrumentation instrumentation){this.instrumentation = instrumentation; classFileTransformer = new ClassFileTransformer() { @Overridepublicbyte[] transform(ClassLoader loader, String className, Class<?> classBeingRedefined, ProtectionDomain protectionDomain, byte[] classfileBuffer) throws IllegalClassFormatException {// ...// TraceTransformerfor (ClassFileTransformer classFileTransformer : traceTransformers) {byte[] transformResult = classFileTransformer.transform(loader, className, classBeingRedefined,protectionDomain, classfileBuffer);if (transformResult != null) {classfileBuffer = transformResult; } }return classfileBuffer; } }; instrumentation.addTransformer(classFileTransformer, true); }

Metachunk* SpaceManager::get_new_chunk(size_t chunk_word_size) {// Get a chunk from the chunk freelist Metachunk* next = chunk_manager()->chunk_freelist_allocate(chunk_word_size);if (next == NULL) {next = vs_list()->get_new_chunk(chunk_word_size, medium_chunk_bunch()); } Log(gc, metaspace, alloc) log;if (log.is_trace() && next != NULL && SpaceManager::is_humongous(next->word_size())) {log.trace(" new humongous chunk word size " PTR_FORMAT, next->word_size()); }returnnext;}

// We have entries mapped between the newand merged constant pools// so we have to rewrite some constant pool references.// 存在需要在新的及合并后的Constant Pool间映射的Entry，因此我们必须重写一些Constant Pool的引用。if (!rewrite_cp_refs(scratch_class, THREAD)) {return JVMTI_ERROR_INTERNAL; }

// Rewrite constant pool references in the specific method. This code// was adapted from Rewriter::rewrite_method().void VM_RedefineClasses::rewrite_cp_refs_in_method(methodHandle method,methodHandle *new_method_p, TRAPS) {// ...// the new value needs ldc_w instead of ldc u_char inst_buffer[4]; // max instruction size is4 bytesbcp = (address)inst_buffer;// construct new instruction sequence *bcp = Bytecodes::_ldc_w; bcp++; Bytes::put_Java_u2(bcp, new_index);Relocator rc(method, NULL /* no RelocatorListener needed */);methodHandle m; {PauseNoSafepointVerifier pnsv(&nsv);// ldc is2 bytes and ldc_w is3 bytes// 执行到这一句进入空间分配 m = rc.insert_space_at(bci, 3, inst_buffer, CHECK); }// return the new method so that the caller can update// the containing class *new_method_p = method = m;//switch our bytecode processing loop from the old method// to the new method// ... } // end we need ldc_w instead of ldc } // end if there is a mapped index } break;// ...

// src/hotspot/share/classfile/classFileParser.cpp # parse_stackmap_tablestatic const u1* parse_stackmap_table(const ClassFileStream* const cfs,u4 code_attribute_length,bool need_verify, TRAPS) {// ... // check code_attribute_length first cfs->skip_u1(code_attribute_length, CHECK_NULL);// 关注这一行if (!need_verify && !DumpSharedSpaces) {returnNULL; }return stackmap_table_start;}

jvmtiError VM_RedefineClasses::load_new_class_versions(TRAPS) {// ...for (int i = 0; i < _class_count; i++) { // Create HandleMark so that any handles created while loading newclass// versions are deleted. Constant pools are deallocated while merging// constant poolsHandleMark hm(THREAD); InstanceKlass* the_class = get_ik(_class_defs[i].klass);Symbol* the_class_sym = the_class-&>name(); log_debug(redefine, class, load) ("loading name=%s kind=%d (avail_mem=" UINT64_FORMAT "K)", the_class-&gt;external_name(), _class_load_kind, os::available_memory() &gt;&gt; 10);// 构造了这个ClassFileStream对象↓ClassFileStreamst((u1*)_class_defs[i].class_bytes, _class_defs[i].class_byte_count,"__VM_RedefineClasses__", ClassFileStream::verify); // ...

// Figure out whether we can skip format checking (matching classic VM behavior)if (DumpSharedSpaces) { // 没有启动参数，为false // verify == true means it's a 'remote' class (i.e., non-boot class)// Verification decision is based on BytecodeVerificationRemote flag // for those classes. _need_verify = (stream->need_verify()) ? BytecodeVerificationRemote : BytecodeVerificationLocal; }else {// 走到这个分支 _need_verify = Verifier::should_verify_for(_loader_data->class_loader(), stream->need_verify()); }bool Verifier::should_verify_for(oop class_loader, bool should_verify_class) {return (class_loader == NULL || !should_verify_class) ?BytecodeVerificationLocal : BytecodeVerificationRemote;}

// src/main/java/com/taobao/arthas/core/advisor/Enhancer.java// ...if (matchingClasses != null && !matchingClasses.contains(classBeingRedefined)) {returnnull;}ClassNode classNode = new ClassNode(Opcodes.ASM8);// 在AsmUtils中新增方法，返回处理ClassNode的ClassReader。// 此时这个ClassReader中已经保存了原始类的Constant Pool等信息// 保持着这个ClassReader对象，在最后生成字节码的时候有用ClassReader classReader = AsmUtils.toClassReader(classfileBuffer, classNode);// remove JSR https://github.com/alibaba/arthas/issues/1304classNode = AsmUtils.removeJSRInstructions(classNode);// 生成增强字节码DefaultInterceptorClassParser defaultInterceptorClassParser = new DefaultInterceptorClassParser();// ...

// src/main/java/com/taobao/arthas/core/advisor/Enhancer.java// ...// https://github.com/alibaba/arthas/issues/1223if (classNode.version<Opcodes.V1_5) { classNode.version = Opcodes.V1_5;}byte[] enhanceClassByteArray = AsmUtils.toBytes(classNode, inClassLoader, classReader);// 增强成功，记录类classBytesCache.put(classBeingRedefined, newObject());// dump the classdumpClassIfNecessary(className, enhanceClassByteArray, affect);// 成功计数affect.cCnt(1);// ...

// src/main/java/com/taobao/arthas/bytekit/utils/AsmUtils.java// ...// 新增方法如下publicstaticbyte[] toBytes(ClassNode classNode, ClassLoader classLoader, ClassReader classReader) {int flags = ClassWriter.COMPUTE_FRAMES | ClassWriter.COMPUTE_MAXS; ClassWriter writer = new ClassLoaderAwareClassWriter(classReader, flags, classLoader);classNode.accept(writer);return writer.toByteArray();}