Skip to content

Commit b8c656a

Browse files
cgivrecgivre
committed
More mods
1 parent 08b599d commit b8c656a

2 files changed

Lines changed: 58 additions & 44 deletions

File tree

‎contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/HiveContainer.java‎

Lines changed: 18 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -94,12 +94,13 @@ private void configureContainer() {
9494
waitingFor(Wait.forLogMessage(".*Hive container ready \\(pre-initialized\\)!.*", 1)
9595
.withStartupTimeout(Duration.ofMinutes(2)));
9696
} else if (useFallbackImage) {
97-
// Fallback to apache/hive:3.1.3 - wait for both metastore port AND HiveServer2 to be ready
98-
// The metastore must be accepting connections before Drill can use it
97+
// apache/hive:3.1.3 official image - wait for HiveServer2 to be fully started
98+
// The official image outputs "HiveServer2 Web UI available at..." when ready
99+
// or we can wait for the listening port plus a reasonable log message
99100
WaitAllStrategy waitStrategy = new WaitAllStrategy()
100-
.withStrategy(Wait.forListeningPort()) // Wait for metastore port 9083 to be listening
101-
.withStrategy(Wait.forLogMessage(".*Starting HiveServer2.*", 1))
102-
.withStartupTimeout(Duration.ofMinutes(10)); // Allow up to 10 minutes for full startup
101+
.withStrategy(Wait.forListeningPort())
102+
.withStrategy(Wait.forLogMessage(".*HiveServer2.*", 1))
103+
.withStartupTimeout(Duration.ofMinutes(15)); // Official image can be slow
103104
waitingFor(waitStrategy);
104105
} else {
105106
// Custom image: wait for metastore port AND test data initialization message
@@ -121,9 +122,8 @@ private void configureContainer() {
121122
/**
122123
* Gets the singleton instance of HiveContainer.
123124
* Container is started on first access and reused for all subsequent tests.
124-
* If the custom image is not available, falls back to apache/hive:3.1.3.
125125
*
126-
* @return Shared HiveContainer instance, or null if Docker is unavailable
126+
* @return Shared HiveContainer instance
127127
* @throws RuntimeException if container fails to start
128128
*/
129129
public static synchronized HiveContainer getInstance() {
@@ -141,25 +141,19 @@ public static synchronized HiveContainer getInstance() {
141141

142142
System.out.println("========================================");
143143
System.out.println("Starting Hive Docker container...");
144-
System.out.println("Requested image: " + HIVE_IMAGE);
144+
System.out.println("Image: " + HIVE_IMAGE);
145145
System.out.println("========================================");
146146

147-
// Try the requested image first
148147
try {
149-
instance = tryStartContainer(HIVE_IMAGE, false);
148+
instance = tryStartContainer(HIVE_IMAGE, HIVE_IMAGE.equals(FALLBACK_IMAGE));
150149
return instance;
151150
} catch (Exception e) {
152-
logger.warn("Failed to start container with image '{}': {}", HIVE_IMAGE, e.getMessage());
153-
System.out.println("Failed to start with " + HIVE_IMAGE + ", trying fallback image...");
154-
}
155-
156-
// Fall back to apache/hive:3.1.3
157-
System.out.println("Falling back to: " + FALLBACK_IMAGE);
158-
try {
159-
instance = tryStartContainer(FALLBACK_IMAGE, true);
160-
return instance;
161-
} catch (Exception e) {
162-
initializationError = "Failed to start Hive container with both custom and fallback images: " + e.getMessage();
151+
initializationError = "Failed to start Hive container with image '" + HIVE_IMAGE + "': " + e.getMessage() +
152+
"\n\nTo build the required Docker image, run:" +
153+
"\n cd contrib/storage-hive/core/src/test/resources/docker" +
154+
"\n docker build -f Dockerfile.fast -t drill-hive-test:fast ." +
155+
"\n\nOr use the public Apache Hive image (slower startup):" +
156+
"\n mvn test -Dhive.image=apache/hive:3.1.3";
163157
logger.error(initializationError, e);
164158
throw new RuntimeException(initializationError, e);
165159
}
@@ -170,16 +164,16 @@ private static HiveContainer tryStartContainer(String imageName, boolean isFallb
170164
if (usePreinit) {
171165
System.out.println("Using pre-initialized image (~1 minute startup)");
172166
} else if (isFallback) {
173-
System.out.println("Using fallback apache/hive image (~5 minute startup)");
167+
System.out.println("Using apache/hive official image (~10-15 minute startup)");
174168
} else {
175-
System.out.println("Using custom image (~15 minute startup on first run)");
169+
System.out.println("Using custom image (~3-5 minute startup on first run)");
176170
}
177-
System.out.println("Image: " + imageName);
178171

179172
logger.info("Creating Hive container with image: {}", imageName);
180173
HiveContainer container = new HiveContainer(imageName, isFallback);
181174

182175
System.out.println("Pulling Docker image and starting container...");
176+
System.out.println("This may take several minutes on first run...");
183177
long startTime = System.currentTimeMillis();
184178
container.start();
185179

@@ -195,9 +189,6 @@ private static HiveContainer tryStartContainer(String imageName, boolean isFallb
195189
System.out.println("Metastore: " + container.getMetastoreUri());
196190
System.out.println("JDBC: " + container.getJdbcUrl());
197191
System.out.println("Container will be reused for all tests");
198-
if (isFallback) {
199-
System.out.println("NOTE: Using fallback image - test data must be created via JDBC");
200-
}
201192
System.out.println("========================================");
202193
logger.info("Hive container started and ready for tests");
203194

‎contrib/storage-hive/core/src/test/resources/docker/entrypoint-fast.sh‎

Lines changed: 40 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -38,16 +38,32 @@ echo "Starting Hive Metastore on port 9083..."
3838
METASTORE_PID=$!
3939
echo "Metastore started (PID: $METASTORE_PID)"
4040

41-
# Wait for metastore to be ready (simple time-based wait + log check)
41+
# Wait for metastore to be ready
4242
echo "Waiting for Metastore to be ready..."
43-
sleep 30
44-
if grep -q "Starting Hive Metastore Server" /tmp/metastore.log; then
45-
echo "✓ Metastore is starting"
46-
else
47-
echo "ERROR: Metastore failed to start"
48-
cat /tmp/metastore.log
49-
exit 1
50-
fi
43+
MAX_WAIT=120 # 2 minutes max
44+
WAIT_COUNT=0
45+
while [ $WAIT_COUNT -lt $MAX_WAIT ]; do
46+
WAIT_COUNT=$((WAIT_COUNT+5))
47+
# Check for various success indicators in the log
48+
if grep -qE "(Starting Hive Metastore Server|Started the new metaserver|Metastore.*started)" /tmp/metastore.log 2>/dev/null; then
49+
echo "Metastore is starting"
50+
break
51+
fi
52+
# Also check if the process is still running
53+
if ! kill -0 $METASTORE_PID 2>/dev/null; then
54+
echo "ERROR: Metastore process died"
55+
cat /tmp/metastore.log
56+
exit 1
57+
fi
58+
if [ $WAIT_COUNT -ge $MAX_WAIT ]; then
59+
echo "WARNING: Could not confirm metastore startup from logs, continuing anyway..."
60+
echo "Metastore log:"
61+
tail -20 /tmp/metastore.log
62+
fi
63+
sleep 5
64+
done
65+
# Extra wait for metastore to fully initialize
66+
sleep 10
5167

5268
# Start HiveServer2 in background
5369
echo "Starting HiveServer2 on port 10000..."
@@ -57,23 +73,30 @@ echo "HiveServer2 started (PID: $HIVESERVER2_PID)"
5773

5874
# Wait for HiveServer2 to accept JDBC connections
5975
echo "Waiting for HiveServer2 to accept JDBC connections..."
60-
echo "This should take 1-3 minutes..."
61-
MAX_RETRIES=60 # 60 attempts × 5 seconds = 5 minutes max
76+
echo "This may take 3-10 minutes..."
77+
MAX_RETRIES=120 # 120 attempts x 5 seconds = 10 minutes max
6278
RETRY_COUNT=0
6379
while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
6480
RETRY_COUNT=$((RETRY_COUNT+1))
6581
if beeline -u jdbc:hive2://localhost:10000 -e "show databases;" > /dev/null 2>&1; then
66-
echo "✓ HiveServer2 is ready and accepting connections!"
82+
echo "HiveServer2 is ready and accepting connections!"
6783
break
6884
fi
69-
if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then
70-
echo "ERROR: HiveServer2 failed to accept connections after 5 minutes"
85+
# Check if process is still running
86+
if ! kill -0 $HIVESERVER2_PID 2>/dev/null; then
87+
echo "ERROR: HiveServer2 process died"
7188
cat /tmp/hiveserver2.log
7289
exit 1
7390
fi
74-
# Show progress every 30 seconds
75-
if [ $((RETRY_COUNT % 6)) -eq 0 ]; then
76-
echo "Waiting for HiveServer2... ($((RETRY_COUNT * 5)) seconds elapsed)"
91+
if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then
92+
echo "ERROR: HiveServer2 failed to accept connections after 10 minutes"
93+
echo "HiveServer2 log:"
94+
tail -50 /tmp/hiveserver2.log
95+
exit 1
96+
fi
97+
# Show progress every minute
98+
if [ $((RETRY_COUNT % 12)) -eq 0 ]; then
99+
echo "Waiting for HiveServer2... ($((RETRY_COUNT * 5 / 60)) minutes elapsed)"
77100
fi
78101
sleep 5
79102
done

0 commit comments

Comments
 (0)