FIR: introduce diagnostic NESTED_CLASS_NOT_ALLOWED

compiler/fir/analysis-tests/testData/resolve/diagnostics/localAnnotationClass.kt

@@ -3,6 +3,6 @@ fun foo() {
 
     @Ann class Local {
         <!LOCAL_ANNOTATION_CLASS_ERROR{LT}!>// There should also be NESTED_CLASS_NOT_ALLOWED report here.
-        <!LOCAL_ANNOTATION_CLASS_ERROR{PSI}!>annotation class Nested<!><!>
+        <!LOCAL_ANNOTATION_CLASS_ERROR{PSI}!>annotation <!NESTED_CLASS_NOT_ALLOWED!>class Nested<!><!><!>
     }
 }

compiler/fir/analysis-tests/testData/resolve/diagnostics/localEntitytNotAllowed.kt

@@ -8,7 +8,7 @@ object A {
     val a = object : Any() {
         <!LOCAL_OBJECT_NOT_ALLOWED!>object D<!> {
             <!LOCAL_OBJECT_NOT_ALLOWED!>object G<!>
-            <!LOCAL_INTERFACE_NOT_ALLOWED!>interface Z<!>
+            <!LOCAL_INTERFACE_NOT_ALLOWED, NESTED_CLASS_NOT_ALLOWED!>interface Z<!>
         }
 
         <!LOCAL_INTERFACE_NOT_ALLOWED!>interface Y<!>
@@ -17,7 +17,7 @@ object A {
     fun b() {
         <!LOCAL_OBJECT_NOT_ALLOWED!>object E<!> {
             <!LOCAL_OBJECT_NOT_ALLOWED!>object F<!>
-            <!LOCAL_INTERFACE_NOT_ALLOWED!>interface M<!>
+            <!LOCAL_INTERFACE_NOT_ALLOWED, NESTED_CLASS_NOT_ALLOWED!>interface M<!>
         }
 
         <!LOCAL_INTERFACE_NOT_ALLOWED!>interface N<!>

compiler/fir/checkers/checkers-component-generator/src/org/jetbrains/kotlin/fir/checkers/generator/diagnostics/FirDiagnosticsList.kt

@@ -42,6 +42,9 @@ object DIAGNOSTICS_LIST : DiagnosticList() {
         val VARIABLE_EXPECTED by error<FirSourceElement, PsiElement>()
         val RETURN_NOT_ALLOWED by error<FirSourceElement, PsiElement>()
         val DELEGATION_IN_INTERFACE by error<FirSourceElement, PsiElement>()
+        val NESTED_CLASS_NOT_ALLOWED by error<FirSourceElement, KtNamedDeclaration>(PositioningStrategy.DECLARATION_NAME) {
+            parameter<String>("declaration")
+        }
     }
 
     val UNRESOLVED by object : DiagnosticGroup("Unresolved") {

compiler/fir/checkers/gen/org/jetbrains/kotlin/fir/analysis/diagnostics/FirErrors.kt

@@ -61,6 +61,7 @@ object FirErrors {
     val VARIABLE_EXPECTED by error0<FirSourceElement, PsiElement>()
     val RETURN_NOT_ALLOWED by error0<FirSourceElement, PsiElement>()
     val DELEGATION_IN_INTERFACE by error0<FirSourceElement, PsiElement>()
+    val NESTED_CLASS_NOT_ALLOWED by error1<FirSourceElement, KtNamedDeclaration, String>(SourceElementPositioningStrategies.DECLARATION_NAME)
 
     // Unresolved
     val HIDDEN by error1<FirSourceElement, PsiElement, AbstractFirBasedSymbol<*>>(SourceElementPositioningStrategies.REFERENCE_BY_QUALIFIED)

compiler/fir/checkers/src/org/jetbrains/kotlin/fir/analysis/checkers/declaration/FirNestedClassChecker.kt

+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.fir.analysis.checkers.declaration
+
+import org.jetbrains.kotlin.descriptors.ClassKind
+import org.jetbrains.kotlin.fir.analysis.checkers.context.CheckerContext
+import org.jetbrains.kotlin.fir.analysis.diagnostics.DiagnosticReporter
+import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.NESTED_CLASS_NOT_ALLOWED
+import org.jetbrains.kotlin.fir.analysis.diagnostics.reportOn
+import org.jetbrains.kotlin.fir.declarations.*
+
+// No need to visit anonymous object since an anonymous object is always inner. This aligns with
+// compiler/frontend/src/org/jetbrains/kotlin/resolve/ModifiersChecker.java:198
+object FirNestedClassChecker : FirRegularClassChecker() {
+    override fun check(declaration: FirRegularClass, context: CheckerContext, reporter: DiagnosticReporter) {
+        // Local enums / objects / companion objects are handled with different diagnostic codes.
+        if ((declaration.classKind.isSingleton || declaration.classKind == ClassKind.ENUM_CLASS) && declaration.isLocal) return
+        val containingDeclaration = context.containingDeclarations.lastOrNull() ?: return
+
+        when (containingDeclaration) {
+            is FirRegularClass -> {
+                if (!declaration.isInner && (containingDeclaration.isInner || containingDeclaration.isLocal)) {
+                    reporter.reportOn(declaration.source, NESTED_CLASS_NOT_ALLOWED, declaration.description, context)
+                }
+            }
+            is FirClass<*> -> {
+                // Since 1.3, enum entries can contain inner classes only.
+                // Companion objects are reported with code WRONG_MODIFIER_CONTAINING_DECLARATION instead
+                if (containingDeclaration.classKind == ClassKind.ENUM_ENTRY && !declaration.isInner && !declaration.isCompanion) {
+                    reporter.reportOn(declaration.source, NESTED_CLASS_NOT_ALLOWED, declaration.description, context)
+                }
+            }
+        }
+    }
+
+    // Note: here we don't differentiate anonymous object like in FE1.0
+    // (org.jetbrains.kotlin.resolve.ModifiersChecker.DetailedClassKind) because this case has been ruled out in the first place.
+    private val FirRegularClass.description: String
+        get() = when (classKind) {
+            ClassKind.CLASS -> "Class"
+            ClassKind.INTERFACE -> "Interface"
+            ClassKind.ENUM_CLASS -> "Enum class"
+            ClassKind.ENUM_ENTRY -> "Enum entry"
+            ClassKind.ANNOTATION_CLASS -> "Annotation class"
+            ClassKind.OBJECT -> if (this.isCompanion) "Companion object" else "Object"
+        }
+}

compiler/fir/checkers/src/org/jetbrains/kotlin/fir/analysis/diagnostics/FirDefaultErrorMessages.kt

@@ -98,6 +98,7 @@ import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.LOCAL_INTERFACE_N
 import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.LOCAL_OBJECT_NOT_ALLOWED
 import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.MANY_COMPANION_OBJECTS
 import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.MISSING_VAL_ON_ANNOTATION_PARAMETER
+import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.NESTED_CLASS_NOT_ALLOWED
 import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.MULTIPLE_VARARG_PARAMETERS
 import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.NONE_APPLICABLE
 import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.NON_ABSTRACT_FUNCTION_WITH_NO_BODY
@@ -211,6 +212,7 @@ class FirDefaultErrorMessages : DefaultErrorMessages.Extension {
             map.put(VARIABLE_EXPECTED, "Variable expected")
             map.put(RETURN_NOT_ALLOWED, "'return' is not allowed here")
             map.put(DELEGATION_IN_INTERFACE, "Interfaces cannot use delegation")
+            map.put(NESTED_CLASS_NOT_ALLOWED, "{0} is not allowed here", TO_STRING)
 
             // Unresolved
             map.put(HIDDEN, "Symbol {0} is invisible", SYMBOL)

compiler/fir/entrypoint/src/org/jetbrains/kotlin/fir/checkers/CommonDeclarationCheckers.kt

@@ -60,6 +60,7 @@ object CommonDeclarationCheckers : DeclarationCheckers() {
         FirTypeParametersInObjectChecker,
         FirMemberFunctionChecker,
         FirMemberPropertyChecker,
+        FirNestedClassChecker,
     )
 
     override val constructorCheckers: Set<FirConstructorChecker> = setOf(

compiler/fir/raw-fir/psi2fir/src/org/jetbrains/kotlin/fir/builder/RawFirBuilder.kt

@@ -793,7 +793,11 @@ class RawFirBuilder(
         override fun visitClassOrObject(classOrObject: KtClassOrObject, data: Unit): FirElement {
             return withChildClassName(
                 classOrObject.nameAsSafeName,
-                classOrObject.isLocal || classOrObject.getStrictParentOfType<KtEnumEntry>() != null
+                classOrObject.isLocal
+                        // TODO: currently enum entry initializer is represented in FIR as an FirAnonymousObject. Because of this, all
+                        //  nested declarations are now marked local. This causes the FirNestedClassChecker to ignore some invalid programs.
+                        // See KT-45115
+                        || classOrObject.getStrictParentOfType<KtEnumEntry>() != null
             ) {
                 val classKind = when (classOrObject) {
                     is KtObjectDeclaration -> ClassKind.OBJECT

compiler/testData/diagnostics/tests/classObjects/InnerClassAccessThroughEnum_after.fir.kt

@@ -11,7 +11,7 @@ enum class C {
             O_O
         }
 
-        class G
+        <!NESTED_CLASS_NOT_ALLOWED!>class G<!>
     },
 
     E4 {

compiler/testData/diagnostics/tests/classObjects/InnerClassAccessThroughEnum_before.fir.kt

@@ -11,7 +11,7 @@ enum class C {
             O_O
         }
 
-        class G
+        <!NESTED_CLASS_NOT_ALLOWED!>class G<!>
     },
 
     E4 {

compiler/testData/diagnostics/tests/classObjects/InnerClassClassObject.fir.kt

@@ -2,7 +2,7 @@
 
 class A {
     inner class B {
-        companion object { }
+        companion <!NESTED_CLASS_NOT_ALLOWED!>object<!> { }
     }
 }
 

compiler/testData/diagnostics/tests/classObjects/multipleDissallowedDefaultObjects.fir.kt

 class A {
     inner class I {
-        companion object A
+        companion <!NESTED_CLASS_NOT_ALLOWED!>object A<!>
 
-        companion <!MANY_COMPANION_OBJECTS!>object B<!>
+        companion <!MANY_COMPANION_OBJECTS, NESTED_CLASS_NOT_ALLOWED!>object B<!>
 
-        companion <!MANY_COMPANION_OBJECTS!>object C<!>
+        companion <!MANY_COMPANION_OBJECTS, NESTED_CLASS_NOT_ALLOWED!>object C<!>
     }
 }
 

compiler/testData/diagnostics/tests/enum/dontCreatePackageTypeForEnumEntry_after.fir.kt

-// !LANGUAGE: +NestedClassesInEnumEntryShouldBeInner
-
-enum class E {
-    FIRST,
-    SECOND {
-        class A
-    };
-}
-
-val foo: Any.() -> Unit = {}
-
-fun f1() = E.FIRST.foo()
-fun f2() = E.FIRST.(foo)()
-fun f3() = E.SECOND.foo()
-fun f4() = E.SECOND.(foo)()
-fun f5() = E.SECOND.<!UNRESOLVED_REFERENCE!>A<!>()

compiler/testData/diagnostics/tests/enum/dontCreatePackageTypeForEnumEntry_after.kt

+// FIR_IDENTICAL
 // !LANGUAGE: +NestedClassesInEnumEntryShouldBeInner
 
 enum class E {

compiler/testData/diagnostics/tests/enum/dontCreatePackageTypeForEnumEntry_before.fir.kt

@@ -3,7 +3,7 @@
 enum class E {
     FIRST,
     SECOND {
-        class A
+        <!NESTED_CLASS_NOT_ALLOWED!>class A<!>
     };
 }
 

compiler/testData/diagnostics/tests/enum/inner/insideInnerClassNotAllowed.fir.kt

-class A {
-    inner class B {
-        enum class E {
-            ENTRY
-        }
-    }
-}

compiler/testData/diagnostics/tests/enum/inner/insideInnerClassNotAllowed.kt

+// FIR_IDENTICAL
 class A {
     inner class B {
         <!NESTED_CLASS_NOT_ALLOWED!>enum class E<!> {

compiler/testData/diagnostics/tests/enum/inner/twoEnumsInClassObjectAndInnerClass.fir.kt

-class A {
-    companion object {
-        enum class E { ENTRY }  // OK
-    }
-    
-    inner class B {
-        enum class E { ENTRY }
-    }
-}

compiler/testData/diagnostics/tests/enum/inner/twoEnumsInClassObjectAndInnerClass.kt

+// FIR_IDENTICAL
 class A {
     companion object {
         enum class E { ENTRY }  // OK

compiler/testData/diagnostics/tests/inner/annotationInInnerClass.fir.kt

 class Outer {
     inner class Inner {
-        annotation class TestNestedAnnotation
+        annotation <!NESTED_CLASS_NOT_ALLOWED!>class TestNestedAnnotation<!>
     }
 }

compiler/testData/diagnostics/tests/inner/enumInInnerClass.fir.kt

 class Outer {
     inner class Inner {
-        enum class TestNestedEnum
+        <!NESTED_CLASS_NOT_ALLOWED!>enum class TestNestedEnum<!>
     }
 }

compiler/testData/diagnostics/tests/inner/innerClassInEnumEntryClass_lv11.fir.kt

@@ -4,9 +4,9 @@ enum class Enum {
     ENTRY_WITH_CLASS {
         inner class TestInner
 
-        class TestNested
+        <!NESTED_CLASS_NOT_ALLOWED!>class TestNested<!>
 
-        interface TestInterface
+        <!NESTED_CLASS_NOT_ALLOWED!>interface TestInterface<!>
 
         object TestObject
 

compiler/testData/diagnostics/tests/inner/innerClassInEnumEntryClass_lv12.fir.kt

@@ -4,9 +4,9 @@ enum class Enum {
     ENTRY_WITH_CLASS {
         inner class TestInner
 
-        class TestNested
+        <!NESTED_CLASS_NOT_ALLOWED!>class TestNested<!>
 
-        interface TestInterface
+        <!NESTED_CLASS_NOT_ALLOWED!>interface TestInterface<!>
 
         object TestObject
 

compiler/testData/diagnostics/tests/inner/innerClassInEnumEntryClass_lv13.fir.kt

@@ -4,9 +4,9 @@ enum class Enum {
     ENTRY_WITH_CLASS {
         inner class TestInner
 
-        class TestNested
+        <!NESTED_CLASS_NOT_ALLOWED!>class TestNested<!>
 
-        interface TestInterface
+        <!NESTED_CLASS_NOT_ALLOWED!>interface TestInterface<!>
 
         object TestObject
 

compiler/testData/diagnostics/tests/inner/interfaceInInnerClass.fir.kt

 class Outer {
     inner class Inner {
-        interface TestNestedInterface
+        <!NESTED_CLASS_NOT_ALLOWED!>interface TestNestedInterface<!>
     }
 }

compiler/testData/diagnostics/tests/inner/nestedClassNotAllowed_after.fir.kt

-// !LANGUAGE: +NestedClassesInEnumEntryShouldBeInner
-
-class A {
-    inner class B {
-        class C
-    }
-    
-    fun foo() {
-        class B {
-            class C
-        }
-    }
-}
-
-fun foo() {
-    class B {
-        class C
-    }
-}
-
-
-enum class E {
-    E1 {
-        class D
-    }
-}

compiler/testData/diagnostics/tests/inner/nestedClassNotAllowed_after.kt

+// FIR_IDENTICAL
 // !LANGUAGE: +NestedClassesInEnumEntryShouldBeInner
 
 class A {

compiler/testData/diagnostics/tests/inner/nestedClassNotAllowed_before.fir.kt

@@ -2,25 +2,25 @@
 
 class A {
     inner class B {
-        class C
+        <!NESTED_CLASS_NOT_ALLOWED!>class C<!>
     }
     
     fun foo() {
         class B {
-            class C
+            <!NESTED_CLASS_NOT_ALLOWED!>class C<!>
         }
     }
 }
 
 fun foo() {
     class B {
-        class C
+        <!NESTED_CLASS_NOT_ALLOWED!>class C<!>
     }
 }
 
 
 enum class E {
     E1 {
-        class D
+        <!NESTED_CLASS_NOT_ALLOWED!>class D<!>
     }
 }

compiler/testData/diagnostics/tests/inner/nestedObject.fir.kt

 // SKIP_TXT
 class Outer {
     inner class Inner1 {
-        object Obj1
+        <!NESTED_CLASS_NOT_ALLOWED!>object Obj1<!>
 
-        companion object Obj2
+        companion <!NESTED_CLASS_NOT_ALLOWED!>object Obj2<!>
 
         inner class Inner2 {
-            object Obj3
+            <!NESTED_CLASS_NOT_ALLOWED!>object Obj3<!>
         }
     }
 }

compiler/testData/diagnostics/tests/localClasses/localAnnotationClass.fir.kt

@@ -4,6 +4,6 @@ fun f() {
     <!LOCAL_ANNOTATION_CLASS_ERROR!>annotation class Anno<!>
 
     @Anno class Local {
-        <!LOCAL_ANNOTATION_CLASS_ERROR!>annotation class Nested<!>
+        <!LOCAL_ANNOTATION_CLASS_ERROR!>annotation <!NESTED_CLASS_NOT_ALLOWED!>class Nested<!><!>
     }
 }

compiler/testData/diagnostics/tests/localClasses/localAnnotationClassError.fir.kt

-// !LANGUAGE: +ProhibitLocalAnnotations
-
-fun f() {
-    <!LOCAL_ANNOTATION_CLASS_ERROR!>annotation class Anno<!>
-
-    @Anno class Local {
-        <!LOCAL_ANNOTATION_CLASS_ERROR!>annotation class Nested<!>
-    }
-}

compiler/testData/diagnostics/tests/localClasses/localAnnotationClassError.kt

+// FIR_IDENTICAL
 // !LANGUAGE: +ProhibitLocalAnnotations
 
 fun f() {

compiler/testData/diagnostics/tests/localInterfaces.fir.kt

@@ -6,7 +6,7 @@ fun foo() {
         <!LOCAL_INTERFACE_NOT_ALLOWED!>interface c<!> {}
     }
     class A {
-        <!LOCAL_INTERFACE_NOT_ALLOWED!>interface d<!> {}
+        <!LOCAL_INTERFACE_NOT_ALLOWED, NESTED_CLASS_NOT_ALLOWED!>interface d<!> {}
     }
     val f = {
         <!LOCAL_INTERFACE_NOT_ALLOWED!>interface e<!> {}

compiler/testData/diagnostics/tests/modifiers/const/applicability.fir.kt

@@ -61,7 +61,7 @@ enum class MyEnum {
 
 class Outer {
     inner class Inner {
-        object C {
+        <!NESTED_CLASS_NOT_ALLOWED!>object C<!> {
             const val a = 18
         }
     }

idea/idea-frontend-fir/idea-fir-low-level-api/src/org/jetbrains/kotlin/idea/fir/low/level/api/file/structure/FileStructureElement.kt

@@ -173,7 +173,10 @@ internal class NonReanalyzableDeclarationStructureElement(
                 firFile,
                 onDeclarationEnter = { firDeclaration ->
                     when {
-                        firDeclaration.isGeneratedDeclaration -> DiagnosticCollectorDeclarationAction.SKIP
+                        // Some generated declaration contains structures that we need to check. For example the FIR representation of an
+                        // enum entry initializer, when present, is a generated anonymous object of kind `ENUM_ENTRY`.
+                        firDeclaration.isGeneratedDeclaration ->
+                            DiagnosticCollectorDeclarationAction.SKIP_CURRENT_DECLARATION_AND_CHECK_NESTED
                         firDeclaration is FirFile -> DiagnosticCollectorDeclarationAction.CHECK_CURRENT_DECLARATION_AND_CHECK_NESTED
                         firDeclaration == fir -> {
                             inCurrentDeclaration = true

idea/idea-frontend-fir/src/org/jetbrains/kotlin/idea/frontend/api/fir/diagnostics/KtFirDataClassConverters.kt

@@ -99,6 +99,13 @@ internal val KT_DIAGNOSTIC_CONVERTER = KtDiagnosticConverterBuilder.buildConvert
             token,
         )
     }
+    add(FirErrors.NESTED_CLASS_NOT_ALLOWED) { firDiagnostic ->
+        NestedClassNotAllowedImpl(
+            firDiagnostic.a,
+            firDiagnostic as FirPsiDiagnostic<*>,
+            token,
+        )
+    }
     add(FirErrors.HIDDEN) { firDiagnostic ->
         HiddenImpl(
             firSymbolBuilder.buildSymbol(firDiagnostic.a.fir as FirDeclaration),

idea/idea-frontend-fir/src/org/jetbrains/kotlin/idea/frontend/api/fir/diagnostics/KtFirDiagnostics.kt

@@ -83,6 +83,11 @@ sealed class KtFirDiagnostic<PSI: PsiElement> : KtDiagnosticWithPsi<PSI> {
         override val diagnosticClass get() = DelegationInInterface::class
     }
 
+    abstract class NestedClassNotAllowed : KtFirDiagnostic<KtNamedDeclaration>() {
+        override val diagnosticClass get() = NestedClassNotAllowed::class
+        abstract val declaration: String
+    }
+
     abstract class Hidden : KtFirDiagnostic<PsiElement>() {
         override val diagnosticClass get() = Hidden::class
         abstract val hidden: KtSymbol

idea/idea-frontend-fir/src/org/jetbrains/kotlin/idea/frontend/api/fir/diagnostics/KtFirDiagnosticsImpl.kt

@@ -114,6 +114,14 @@ internal class DelegationInInterfaceImpl(
     override val firDiagnostic: FirPsiDiagnostic<*> by weakRef(firDiagnostic)
 }
 
+internal class NestedClassNotAllowedImpl(
+    override val declaration: String,
+    firDiagnostic: FirPsiDiagnostic<*>,
+    override val token: ValidityToken,
+) : KtFirDiagnostic.NestedClassNotAllowed(), KtAbstractFirDiagnostic<KtNamedDeclaration> {
+    override val firDiagnostic: FirPsiDiagnostic<*> by weakRef(firDiagnostic)
+}
+
 internal class HiddenImpl(
     override val hidden: KtSymbol,
     firDiagnostic: FirPsiDiagnostic<*>,