Skip to content

Commit

Permalink
Merge pull request #769 from Kotlin/jupyter-integration-module
Browse files Browse the repository at this point in the history
Move Jupyter integration from :core to a new module
  • Loading branch information
koperagen authored Jul 9, 2024
2 parents 2fe9278 + 52306e0 commit bb457bf
Show file tree
Hide file tree
Showing 22 changed files with 136 additions and 43 deletions.
5 changes: 0 additions & 5 deletions core/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ plugins {
alias(kotlin.jvm)
alias(publisher)
alias(serialization)
alias(jupyter.api)
alias(korro)
alias(keywordGenerator)
alias(kover)
Expand Down Expand Up @@ -374,10 +373,6 @@ tasks.test {
}
}

tasks.processJupyterApiResources {
libraryProducers = listOf("org.jetbrains.kotlinx.dataframe.jupyter.Integration")
}

kotlinPublications {
publication {
publicationName.set("core")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package org.jetbrains.kotlinx.dataframe.codeGen

import org.jetbrains.kotlinx.jupyter.api.Code
import org.jetbrains.kotlinx.jupyter.api.VariableName
public typealias Code = String
public typealias VariableName = String

/**
* Class representing generated code declarations for a [Marker].
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ package org.jetbrains.dataframe.impl.codeGen

import org.jetbrains.kotlinx.dataframe.AnyFrame
import org.jetbrains.kotlinx.dataframe.AnyRow
import org.jetbrains.kotlinx.dataframe.codeGen.Code
import org.jetbrains.kotlinx.dataframe.codeGen.CodeWithConverter
import org.jetbrains.kotlinx.dataframe.impl.codeGen.ReplCodeGeneratorImpl
import org.jetbrains.kotlinx.jupyter.api.Code
import kotlin.reflect.KClass
import kotlin.reflect.KProperty

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import org.jetbrains.kotlinx.dataframe.annotations.DataSchema
import org.jetbrains.kotlinx.dataframe.api.ColumnSelectionDsl
import org.jetbrains.kotlinx.dataframe.api.DataSchemaEnum
import org.jetbrains.kotlinx.dataframe.codeGen.BaseField
import org.jetbrains.kotlinx.dataframe.codeGen.Code
import org.jetbrains.kotlinx.dataframe.codeGen.CodeWithConverter
import org.jetbrains.kotlinx.dataframe.codeGen.DefaultReadDfMethod
import org.jetbrains.kotlinx.dataframe.codeGen.ExtensionsCodeGenerator
Expand All @@ -33,7 +34,6 @@ import org.jetbrains.kotlinx.dataframe.codeGen.toNullable
import org.jetbrains.kotlinx.dataframe.columns.ColumnGroup
import org.jetbrains.kotlinx.dataframe.impl.toSnakeCase
import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema
import org.jetbrains.kotlinx.jupyter.api.Code

private fun renderNullability(nullable: Boolean) = if (nullable) "?" else ""

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@ import org.jetbrains.kotlinx.dataframe.DataFrame
import org.jetbrains.kotlinx.dataframe.DataRow
import org.jetbrains.kotlinx.dataframe.annotations.DataSchema
import org.jetbrains.kotlinx.dataframe.api.schema
import org.jetbrains.kotlinx.dataframe.codeGen.Code
import org.jetbrains.kotlinx.dataframe.codeGen.CodeWithConverter
import org.jetbrains.kotlinx.dataframe.codeGen.Marker
import org.jetbrains.kotlinx.dataframe.codeGen.MarkerVisibility
import org.jetbrains.kotlinx.dataframe.codeGen.MarkersExtractor
import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema
import org.jetbrains.kotlinx.jupyter.api.Code
import kotlin.reflect.KClass
import kotlin.reflect.KProperty
import kotlin.reflect.KType
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import org.jetbrains.dataframe.impl.codeGen.CodeGenerator
import org.jetbrains.kotlinx.dataframe.AnyFrame
import org.jetbrains.kotlinx.dataframe.DataFrame
import org.jetbrains.kotlinx.dataframe.api.schema
import org.jetbrains.kotlinx.dataframe.codeGen.Code
import org.jetbrains.kotlinx.dataframe.codeGen.DefaultReadDfMethod
import org.jetbrains.kotlinx.dataframe.io.SupportedCodeGenerationFormat
import org.jetbrains.kotlinx.dataframe.io.SupportedDataFrameFormat
Expand All @@ -12,7 +13,6 @@ import org.jetbrains.kotlinx.dataframe.io.guessFormat
import org.jetbrains.kotlinx.dataframe.io.read
import org.jetbrains.kotlinx.dataframe.io.readCodeForGeneration
import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema
import org.jetbrains.kotlinx.jupyter.api.Code
import java.net.URL

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ import org.jetbrains.kotlinx.dataframe.annotations.ImportDataSchema
import org.jetbrains.kotlinx.dataframe.annotations.Interpretable
import org.jetbrains.kotlinx.dataframe.annotations.OptInRefine
import org.jetbrains.kotlinx.dataframe.api.single
import org.jetbrains.kotlinx.dataframe.codeGen.Code
import org.jetbrains.kotlinx.dataframe.codeGen.DefaultReadDfMethod
import org.jetbrains.kotlinx.jupyter.api.Code
import java.io.BufferedInputStream
import java.io.File
import java.io.FileNotFoundException
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,8 @@
package org.jetbrains.kotlinx.dataframe.jupyter

import org.jetbrains.kotlinx.dataframe.io.DisplayConfiguration
import org.jetbrains.kotlinx.dataframe.io.internallyRenderable
import org.jetbrains.kotlinx.dataframe.io.renderValueForHtml
import org.jetbrains.kotlinx.dataframe.io.tooltipLimit
import org.jetbrains.kotlinx.jupyter.api.MimeTypedResult
import org.jetbrains.kotlinx.jupyter.api.Notebook
import org.jetbrains.kotlinx.jupyter.api.Renderable
import org.jetbrains.kotlinx.jupyter.api.libraries.ExecutionHost

public data class RenderedContent(
val truncatedContent: String,
Expand Down Expand Up @@ -68,23 +63,3 @@ public object DefaultCellRenderer : CellRenderer {
return renderValueForHtml(value, tooltipLimit, configuration.decimalFormat).truncatedContent
}
}

internal class JupyterCellRenderer(
private val notebook: Notebook,
private val host: ExecutionHost,
) : ChainedCellRenderer(DefaultCellRenderer) {
override fun maybeContent(value: Any?, configuration: DisplayConfiguration): RenderedContent? {
val renderersProcessor = notebook.renderersProcessor
if (internallyRenderable(value)) return null
val renderedVal = renderersProcessor.renderValue(host, value)
val finalVal = if (renderedVal is Renderable) renderedVal.render(notebook) else renderedVal
if (finalVal is MimeTypedResult && "text/html" in finalVal) return RenderedContent.media(
finalVal["text/html"] ?: ""
)
return renderValueForHtml(finalVal, configuration.cellContentLimit, configuration.decimalFormat)
}

override fun maybeTooltip(value: Any?, configuration: DisplayConfiguration): String? {
return null
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import org.jetbrains.kotlinx.dataframe.api.group
import org.jetbrains.kotlinx.dataframe.api.into
import org.jetbrains.kotlinx.dataframe.api.parse
import org.jetbrains.kotlinx.dataframe.io.toStandaloneHTML
import org.jetbrains.kotlinx.jupyter.util.findNthSubstring
import org.junit.Ignore
import org.junit.Test
import java.awt.Desktop
Expand Down Expand Up @@ -38,4 +37,18 @@ class HtmlRenderingTests : BaseTest() {
html shouldContain "href"
html.findNthSubstring(address, 2) shouldNotBe -1
}

private fun String.findNthSubstring(s: String, n: Int, start: Int = 0): Int {
if (n < 1 || start == -1) return -1

var i = start

for (k in 1..n) {
i = indexOf(s, i)
if (i == -1) return -1
i += s.length
}

return i - s.length
}
}
60 changes: 60 additions & 0 deletions dataframe-jupyter/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
plugins {
alias(libs.plugins.kotlin.jvm)
alias(libs.plugins.publisher)
alias(libs.plugins.jupyter.api)
}

group = "org.jetbrains.kotlinx"

repositories {
mavenCentral()
}

dependencies {
compileOnly(project(":core"))
testImplementation(libs.junit)
testImplementation(libs.serialization.json)
testImplementation(project(":core"))
testImplementation(libs.kotestAssertions) {
exclude("org.jetbrains.kotlin", "kotlin-stdlib-jdk8")
}
}

kotlin {
explicitApi()
}

tasks.compileKotlin {
kotlinOptions {
freeCompilerArgs += listOf("-Xfriend-paths=${project(":core").projectDir}")
jvmTarget = "11"
}
}

tasks.compileTestKotlin {
kotlinOptions {
val friendModule = project(":core")
val jarTask = friendModule.tasks.getByName("jar") as Jar
val jarPath = jarTask.archiveFile.get().asFile.absolutePath
freeCompilerArgs += "-Xfriend-paths=$jarPath"
jvmTarget = "11"
}
}

tasks.withType<JavaCompile> {
sourceCompatibility = JavaVersion.VERSION_11.toString()
targetCompatibility = JavaVersion.VERSION_11.toString()
}

tasks.processJupyterApiResources {
libraryProducers = listOf("org.jetbrains.kotlinx.dataframe.jupyter.Integration")
}

kotlinPublications {
publication {
publicationName.set("dataframe-jupyter")
artifactId.set("dataframe-jupyter")
description.set("Kotlin DataFrame integration with Kotlin Jupyter")
packageName.set("dataframe-jupyter")
}
}
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
@file:Suppress("INVISIBLE_REFERENCE", "INVISIBLE_MEMBER")

package org.jetbrains.kotlinx.dataframe.jupyter

import org.jetbrains.dataframe.impl.codeGen.CodeGenerator
Expand Down Expand Up @@ -32,7 +34,6 @@ import org.jetbrains.kotlinx.dataframe.api.asColumnGroup
import org.jetbrains.kotlinx.dataframe.api.asDataFrame
import org.jetbrains.kotlinx.dataframe.api.columnsCount
import org.jetbrains.kotlinx.dataframe.api.isColumnGroup
import org.jetbrains.kotlinx.dataframe.api.name
import org.jetbrains.kotlinx.dataframe.codeGen.CodeWithConverter
import org.jetbrains.kotlinx.dataframe.columns.ColumnGroup
import org.jetbrains.kotlinx.dataframe.columns.ColumnReference
Expand Down Expand Up @@ -230,10 +231,10 @@ internal class Integration(
{ "DataRow: index = ${it.index()}, columnsCount = ${it.columnsCount()}" },
)
render<ColumnGroup<*>>(
{ """ColumnGroup: name = "${it.name}", rowsCount = ${it.rowsCount()}, columnsCount = ${it.columnsCount()}""" },
{ """ColumnGroup: name = "${it.name()}", rowsCount = ${it.rowsCount()}, columnsCount = ${it.columnsCount()}""" },
)
render<AnyCol>(
{ """DataColumn: name = "${it.name}", type = ${renderType(it.type())}, size = ${it.size()}""" },
{ """DataColumn: name = "${it.name()}", type = ${renderType(it.type())}, size = ${it.size()}""" },
)
render<AnyFrame>(
{ "DataFrame: rowsCount = ${it.rowsCount()}, columnsCount = ${it.columnsCount()}" }
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
@file:Suppress("INVISIBLE_REFERENCE", "INVISIBLE_MEMBER")

package org.jetbrains.kotlinx.dataframe.jupyter

import org.jetbrains.kotlinx.dataframe.io.DisplayConfiguration
import org.jetbrains.kotlinx.dataframe.io.internallyRenderable
import org.jetbrains.kotlinx.dataframe.io.renderValueForHtml
import org.jetbrains.kotlinx.jupyter.api.MimeTypedResult
import org.jetbrains.kotlinx.jupyter.api.Notebook
import org.jetbrains.kotlinx.jupyter.api.Renderable
import org.jetbrains.kotlinx.jupyter.api.libraries.ExecutionHost

internal class JupyterCellRenderer(
private val notebook: Notebook,
private val host: ExecutionHost,
) : ChainedCellRenderer(DefaultCellRenderer) {
override fun maybeContent(value: Any?, configuration: DisplayConfiguration): RenderedContent? {
val renderersProcessor = notebook.renderersProcessor
if (internallyRenderable(value)) return null
val renderedVal = renderersProcessor.renderValue(host, value)
val finalVal = if (renderedVal is Renderable) renderedVal.render(notebook) else renderedVal
if (finalVal is MimeTypedResult && "text/html" in finalVal) return RenderedContent.media(
finalVal["text/html"] ?: ""
)
return renderValueForHtml(finalVal, configuration.cellContentLimit, configuration.decimalFormat)
}

override fun maybeTooltip(value: Any?, configuration: DisplayConfiguration): String? {
return null
}
}
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
@file:Suppress("INVISIBLE_REFERENCE", "INVISIBLE_MEMBER")

package org.jetbrains.kotlinx.dataframe.jupyter

import kotlinx.serialization.ExperimentalSerializationApi
Expand Down Expand Up @@ -79,8 +81,8 @@ internal inline fun <reified T : Any> JupyterHtmlRenderer.render(
val jsonEncodedDf = when {
!ideBuildNumber.supportsDynamicNestedTables() -> {
buildJsonObject {
put(NROW, df.size.nrow)
put(NCOL, df.size.ncol)
put(NROW, df.size().nrow)
put(NCOL, df.size().ncol)
putJsonArray(COLUMNS) { addAll(df.columnNames()) }
put(KOTLIN_DATAFRAME, encodeFrame(df.take(limit)))
}.toString()
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
@file:Suppress("INVISIBLE_REFERENCE", "INVISIBLE_MEMBER")

package org.jetbrains.kotlinx.dataframe.jupyter

import io.kotest.assertions.throwables.shouldNotThrow
Expand Down
13 changes: 13 additions & 0 deletions dataframe-openapi/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ dependencies {
}

testApi(project(":core"))
testImplementation(project(":dataframe-jupyter"))
testImplementation(libs.junit)
testImplementation(libs.kotestAssertions) {
exclude("org.jetbrains.kotlin", "kotlin-stdlib-jdk8")
Expand All @@ -49,3 +50,15 @@ kotlinPublications {
kotlin {
explicitApi()
}

// Only for tests. Because tests depend on kotlin-jupyter, and it's compatible with Java 11
tasks.compileTestJava {
sourceCompatibility = JavaVersion.VERSION_11.toString()
targetCompatibility = JavaVersion.VERSION_11.toString()
}

tasks.compileTestKotlin {
kotlinOptions {
jvmTarget = "11"
}
}
1 change: 1 addition & 0 deletions settings.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -41,3 +41,4 @@ plugins {
}
include("dataframe-excel")
include("core")
include("dataframe-jupyter")

0 comments on commit bb457bf

Please sign in to comment.