提交 5584a80d 编写于 作者: K kohsuke

Adding ZFS migration support, but not activated for now.

git-svn-id: https://hudson.dev.java.net/svn/hudson/trunk/hudson/main@15248 71c3de6d-444a-0410-be80-ed276b4c234a
上级 14ad4216
......@@ -632,6 +632,11 @@ THE SOFTWARE.
<artifactId>libpam4j</artifactId>
<version>1.1</version>
</dependency>
<dependency>
<groupId>org.jvnet.libzfs</groupId>
<artifactId>libzfs</artifactId>
<version>0.2</version>
</dependency>
<!-- offline profiler API to put in the classpath if we need it -->
<!--dependency>
......
/*
* The MIT License
*
* Copyright (c) 2004-2009, Sun Microsystems, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
package hudson.lifecycle;
import com.sun.akuma.Daemon;
import com.sun.akuma.JavaVMArguments;
import hudson.FilePath;
import hudson.Launcher.LocalLauncher;
import hudson.Util;
import hudson.model.AdministrativeMonitor;
import hudson.model.Hudson;
import hudson.model.TaskListener;
import hudson.util.HudsonIsRestarting;
import hudson.util.StreamTaskListener;
import hudson.util.ForkOutputStream;
import static hudson.util.jna.GNUCLibrary.*;
import org.jvnet.solaris.libzfs.LibZFS;
import org.jvnet.solaris.libzfs.ZFSFileSystem;
import org.jvnet.solaris.libzfs.ZFSPool;
import org.jvnet.solaris.libzfs.ZFSType;
import org.jvnet.solaris.mount.MountFlags;
import org.kohsuke.stapler.StaplerRequest;
import org.kohsuke.stapler.StaplerResponse;
import org.apache.commons.io.output.ByteArrayOutputStream;
import javax.servlet.ServletException;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* Encourages the user to migrate HUDSON_HOME on a ZFS file system.
*
* @author Kohsuke Kawaguchi
* @since 1.283
*/
public class ZFSInstaller extends AdministrativeMonitor {
/**
* True if $HUDSON_HOME is a ZFS file system by itself.
*/
private final boolean active = shouldBeActive();
/**
* This will be the file system name that we'll create.
*/
private String prospectiveZfsFileSystemName;
public boolean isActivated() {
return active;
}
public boolean isRoot() {
return LIBC.geteuid()==0;
}
public String getProspectiveZfsFileSystemName() {
return prospectiveZfsFileSystemName;
}
private boolean shouldBeActive() {
if(!System.getProperty("os.name").equals("SunOS"))
// on systems that don't have ZFS, we don't need this monitor
return false;
try {
LibZFS zfs = new LibZFS();
List<ZFSPool> roots = zfs.roots();
if(roots.isEmpty())
return false; // no active ZFS pool
// if we don't run on a ZFS file system, activate
ZFSFileSystem hudsonZfs = zfs.getFileSystemByMountPoint(Hudson.getInstance().getRootDir());
if(hudsonZfs!=null)
return false; // already on ZFS
// decide what file system we'll create
ZFSPool pool = roots.get(0);
prospectiveZfsFileSystemName = computeHudsonFileSystemName(zfs,pool);
return true;
} catch (Exception e) {
LOGGER.log(Level.WARNING, "Failed to detect whether Hudson is on ZFS",e);
return false;
}
}
/**
* Called from the management screen.
*/
public void doAct(StaplerRequest req, StaplerResponse rsp) throws ServletException, IOException {
requirePOST();
Hudson.getInstance().checkPermission(Hudson.ADMINISTER);
if(req.hasParameter("n")) {
// we'll shut up
disable(true);
rsp.sendRedirect2(req.getContextPath());
return;
}
rsp.sendRedirect2("confirm");
}
/**
* Called from the confirmation screen to actually initiate the migration.
*/
public void doStart(StaplerRequest req, StaplerResponse rsp) throws ServletException, IOException {
requirePOST();
Hudson hudson = Hudson.getInstance();
hudson.checkPermission(Hudson.ADMINISTER);
hudson.servletContext.setAttribute("app",new HudsonIsRestarting());
// redirect the user to the manage page
rsp.sendRedirect2(req.getContextPath()+"/manage");
// asynchronously restart, so that we can give a bit of time to the browser to load "restarting..." screen.
new Thread("restart thread") {
@Override
public void run() {
try {
Thread.sleep(5000);
// close all descriptors on exec except stdin,out,err
int sz = LIBC.getdtablesize();
for(int i=3; i<sz; i++) {
int flags = LIBC.fcntl(i, F_GETFD);
if(flags<0) continue;
LIBC.fcntl(i, F_SETFD,flags| FD_CLOEXEC);
}
JavaVMArguments args = JavaVMArguments.current();
args.setSystemProperty(ZFSInstaller.class.getName(),"migrate");
Daemon.selfExec(args);
} catch (InterruptedException e) {
LOGGER.log(Level.SEVERE, "Restart failed",e);
} catch (IOException e) {
LOGGER.log(Level.SEVERE, "Restart failed",e);
}
}
}.start();
}
public static void init() {
List<AdministrativeMonitor> monitors = Hudson.getInstance().administrativeMonitors;
if("migrate".equals(System.getProperty(ZFSInstaller.class.getName()))) {
ByteArrayOutputStream out = new ByteArrayOutputStream();
StreamTaskListener listener = new StreamTaskListener(new ForkOutputStream(System.out, out));
try {
if(migrate(listener)) {
// completed successfully
monitors.add(new MigrationCompleteNotice());
return;
}
} catch (IOException e) {
e.printStackTrace(listener.error("Migration failed"));
} catch (InterruptedException e) {
e.printStackTrace(listener.error("Migration failed"));
}
// migration failed
monitors.add(new MigrationFailedNotice(out));
return;
}
if(System.getProperty(ZFSInstaller.class.getName())!=null)
// this feature is still being tested, so only activate this during a test
monitors.add(new ZFSInstaller());
}
/**
* Migrates $HUDSON_HOME to a new ZFS file system.
*
* TODO: do this in a separate JVM to elevate the privilege.
*
* @param listener
* Log of migration goes here.
* @return
* false if a migration failed.
*/
private static boolean migrate(TaskListener listener) throws IOException, InterruptedException {
PrintStream out = listener.getLogger();
File home = Hudson.getInstance().getRootDir();
// do the migration
LibZFS zfs = new LibZFS();
ZFSFileSystem existing = zfs.getFileSystemByMountPoint(home);
if(existing!=null) {
out.println(home+" is already on ZFS. Doing nothing");
return true;
}
File tmpDir = Util.createTempDir();
// mount a new file system to a temporary location
String name = computeHudsonFileSystemName(zfs, zfs.roots().get(0));
out.println("Creating "+name);
ZFSFileSystem hudson = (ZFSFileSystem)zfs.create(name, ZFSType.FILESYSTEM);
hudson.setMountPoint(tmpDir);
hudson.mount();
// copy all the files
out.println("Copying all existing data files");
if(system(home,listener, "/usr/bin/cp","-pR",".", tmpDir.getAbsolutePath())!=0) {
out.println("Failed to copy "+home+" to "+tmpDir);
return false;
}
// unmount
out.println("Unmounting "+name);
hudson.unmount(MountFlags.MS_FORCE);
// move the original directory to the side
File backup = new File(home.getPath()+".backup");
out.println("Moving "+home+" to "+backup);
if(!home.renameTo(backup)) {
out.println("Failed to move your current data "+home+" out of the way");
}
// update the mount point
out.println("Creating a new mount point at "+home);
if(!home.mkdir())
throw new IOException("Failed to create mount point "+home);
out.println("Mounting "+name);
hudson.setMountPoint(home);
hudson.mount();
// delete back up
out.println("Deleting "+backup);
if(system(new File("/"),listener,"/usr/bin/rm","-rf",backup.getAbsolutePath())!=0) {
out.println("Failed to delete "+home+" to "+tmpDir);
return false;
}
out.println("Migration completed");
return true;
}
private static int system(File pwd, TaskListener listener, String... args) throws IOException, InterruptedException {
return new LocalLauncher(listener).launch(args, new String[0], System.out, new FilePath(pwd)).join();
}
private static String computeHudsonFileSystemName(LibZFS zfs, ZFSPool pool) {
if(!zfs.exists(pool.getName()+"/hudson"))
return pool.getName()+"/hudson";
for( int i=2; ; i++ ) {
String name = pool.getName() + "/hudson" + i;
if(!zfs.exists(name))
return name;
}
}
/**
* Used to indicate that the migration was completed successfully.
*/
public static final class MigrationCompleteNotice extends AdministrativeMonitor {
public boolean isActivated() {
return true;
}
}
/**
* Used to indicate a failure in the migration.
*/
public static final class MigrationFailedNotice extends AdministrativeMonitor {
ByteArrayOutputStream record;
MigrationFailedNotice(ByteArrayOutputStream record) {
this.record = record;
}
public boolean isActivated() {
return true;
}
public String getLog() {
return record.toString();
}
}
private static final Logger LOGGER = Logger.getLogger(ZFSInstaller.class.getName());
}
......@@ -43,6 +43,7 @@ import hudson.UDPBroadcastThread;
import hudson.logging.LogRecorderManager;
import hudson.lifecycle.WindowsInstallerLink;
import hudson.lifecycle.Lifecycle;
import hudson.lifecycle.ZFSInstaller;
import hudson.model.Descriptor.FormException;
import hudson.model.listeners.ItemListener;
import hudson.model.listeners.JobListener;
......@@ -470,6 +471,7 @@ public final class Hudson extends AbstractModelObject implements ItemGroup<TopLe
// run the init code of SubversionSCM before we load plugins so that plugins can change SubversionWorkspaceSelector.
SubversionSCM.DescriptorImpl.DESCRIPTOR.getDisplayName();
ZFSInstaller.init();
// load plugins.
pluginManager = new PluginManager(context);
......@@ -1200,6 +1202,16 @@ public final class Hudson extends AbstractModelObject implements ItemGroup<TopLe
}
}
/**
* Binds {@link AdministrativeMonitor}s to URL.
*/
public AdministrativeMonitor getAdministrativeMonitor(String id) {
for (AdministrativeMonitor m : administrativeMonitors)
if(m.id.equals(id))
return m;
return null;
}
public NodeDescriptor getDescriptor() {
return DescriptorImpl.INSTANCE;
}
......
......@@ -43,12 +43,23 @@ public interface GNUCLibrary extends Library {
int geteuid();
int getppid();
int chdir(String dir);
int getdtablesize();
int execv(String file, StringArray args);
int setenv(String name, String value);
int unsetenv(String name);
void perror(String msg);
String strerror(int errno);
int fcntl(int fd, int command);
int fcntl(int fd, int command, int flags);
// obtained from Linux. Needs to be checked if these values are portable.
static final int F_GETFD = 1;
static final int F_SETFD = 2;
static final int FD_CLOEXEC = 1;
// this is listed in http://developer.apple.com/DOCUMENTATION/Darwin/Reference/ManPages/man3/sysctlbyname.3.html
// but not in http://www.gnu.org/software/libc/manual/html_node/System-Parameters.html#index-sysctl-3493
// perhaps it is only supported on BSD?
......
<!--
The MIT License
Copyright (c) 2004-2009, Sun Microsystems, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
-->
<j:jelly xmlns:j="jelly:core" xmlns:st="jelly:stapler" xmlns:d="jelly:define" xmlns:l="/lib/layout" xmlns:t="/lib/hudson" xmlns:f="/lib/form">
<div class="info">
<form method="post" action="${rootURL}/${it.url}/disable">
${%Data was successfully migrated to ZFS.}
<f:submit value="${%OK}"/>
</form>
</div>
</j:jelly>
\ No newline at end of file
<!--
The MIT License
Copyright (c) 2004-2009, Sun Microsystems, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
-->
<j:jelly xmlns:j="jelly:core" xmlns:st="jelly:stapler" xmlns:d="jelly:define" xmlns:l="/lib/layout" xmlns:t="/lib/hudson" xmlns:f="/lib/form" xmlns:i="jelly:fmt">
<l:layout title="${%ZFS Migration Problem}">
<l:main-panel>
<h1>${%ZFS Migration Problem}</h1>
<pre><st:out value="${it.log}" /></pre>
</l:main-panel>
</l:layout>
</j:jelly>
<!--
The MIT License
Copyright (c) 2004-2009, Sun Microsystems, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
-->
<j:jelly xmlns:j="jelly:core" xmlns:st="jelly:stapler" xmlns:d="jelly:define" xmlns:l="/lib/layout" xmlns:t="/lib/hudson" xmlns:f="/lib/form">
<div class="error">
${%ZFS migration failed.}
<a href="${it.url}/">See the log for more details</a>.
</div>
</j:jelly>
\ No newline at end of file
<!--
The MIT License
Copyright (c) 2004-2009, Sun Microsystems, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
-->
<j:jelly xmlns:j="jelly:core" xmlns:st="jelly:stapler" xmlns:d="jelly:define" xmlns:l="/lib/layout" xmlns:t="/lib/hudson" xmlns:f="/lib/form">
<l:layout title="${%ZFS file system creation}">
<l:header />
<l:side-panel />
<l:main-panel>
<h1>${%ZFS file system creation}</h1>
<div>
${%blurb}
<j:if test="${!it.isRoot()}">
<div class="warning">
${%You'll need the root password of the system to do this.}
</div>
</j:if>
<ol>
<li>${%Restart itself so that the migration can be done without worrying about concurrent data modifications}</li>
<li>${%create(it.prospectiveZfsFileSystemName)}</li>
<li>${%rename(app.rootDir)}</li>
<li>${%mount(app.rootDir)}</li>
<li>${%delete(app.rootDir)}</li>
</ol>
<form action="start" method="post">
<f:submit value="${%Start migration}" />
</form>
</div>
</l:main-panel>
</l:layout>
</j:jelly>
\ No newline at end of file
# The MIT License
#
# Copyright (c) 2004-2009, Sun Microsystems, Inc., Kohsuke Kawaguchi
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
blurb=Hudson will perform the following steps to migrate your existing data to a ZFS file system.
create=Create a new ZFS file system <tt>{0}</tt> and copy all the data into it
rename=Rename <tt>{0}</tt> to <tt>{0}.backup</tt>
mount=Mount a new ZFS file system at <tt>{0}</tt>
delete=Delete <tt>{0}.backup</tt>
\ No newline at end of file
<!--
The MIT License
Copyright (c) 2004-2009, Sun Microsystems, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
-->
<j:jelly xmlns:j="jelly:core" xmlns:st="jelly:stapler" xmlns:d="jelly:define" xmlns:l="/lib/layout" xmlns:t="/lib/hudson" xmlns:f="/lib/form">
<div class="warning">
<form method="post" action="${rootURL}/${it.url}/act">
<p>${%blurb}</p>
<div style="margin-left:2em">
<f:submit name="y" value="${%Yes, please}"/>
<f:submit name="n" value="${%No, thank you}"/>
</div>
</form>
</div>
</j:jelly>
\ No newline at end of file
# The MIT License
#
# Copyright (c) 2004-2009, Sun Microsystems, Inc., Kohsuke Kawaguchi
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
blurb=You are running on Solaris. Would you like Hudson to create a ZFS file system for you \
so that you can get the most out of Solaris?
\ No newline at end of file
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册