xorl %eax, %eax

Solaris dtrace(1M) Kernel Panic

with one comment

This issue was disclosed by Sun on 27 April 2009 and affects Solaris 10 as well as OpenSolaris snv_01 through snv_113. Dtrace is an advanced debugging tool developed by Sun which can be found under usr/src/uts/common/dtrace/ directory in the OpenSolaris’ source code. Here is the first buggy routine:

50 /*
51  * User-Land Trap-Based Tracing
52  * ----------------------------
53  *
54  * The fasttrap provider allows DTrace consumers to instrument any user-level
55  * instruction to gather data; this includes probes with semantic
56  * signifigance like entry and return as well as simple offsets into the
57  * function. While the specific techniques used are very ISA specific, the
58  * methodology is generalizable to any architecture.
       ...
1916 /*ARGSUSED*/
1917 static int
1918 fasttrap_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv)
1919 {
1920    if (!dtrace_attached())
       ...
1923    if (cmd == FASTTRAPIOC_MAKEPROBE) {
1924        fasttrap_probe_spec_t *uprobe = (void *)arg;
1925        fasttrap_probe_spec_t *probe;
       ...
1931        if (copyin(&uprobe->ftps_noffs, &noffs,
1932            sizeof (uprobe->ftps_noffs)))
1933            return (EFAULT);
1934
1935        /*
1936         * Probes must have at least one tracepoint.
1937         */
1938        if (noffs == 0)
1939            return (EINVAL);
1940
1941        size = sizeof (fasttrap_probe_spec_t) +
1942            sizeof (probe->ftps_offs[0]) * (noffs - 1);
1943
1944        if (size > 1024 * 1024)
1945            return (ENOMEM);
1946
1947        probe = kmem_alloc(size, KM_SLEEP);
1948
1949        if (copyin(uprobe, probe, size) != 0) {
1950            kmem_free(probe, size);
1951            return (EFAULT);
1952        }
       ...
1997        ret = fasttrap_add_probe(probe);
       ...
2065    return (EINVAL);
2066 }

This code can be found at /usr/src/uts/common/dtrace/fasttrap.c and as you can see, the handle code for FASTTRAPIOC_MAKEPROBE IOCTL call parses the user controlled argument (line 1924) as a fasttrap_probe_spec_t pointer to structure. This structure is defined at usr/src/uts/common/sys/fasttrap.h like this:

53 typedef struct fasttrap_probe_spec {
54     pid_t            ftps_pid;
55     fasttrap_probe_type_t    ftps_type;
56
57     char            ftps_func[DTRACE_FUNCNAMELEN];
58     char            ftps_mod[DTRACE_MODNAMELEN];
59
60     uint64_t        ftps_pc;
61     uint64_t        ftps_size;
62     uint64_t        ftps_noffs;
63     uint64_t        ftps_offs[1];
64 } fasttrap_probe_spec_t;

Back to the IOCTL call, it checks that there is at least one tracepoint at line 1938 and if this is the case, it calculates the size to allocate. If that size results in a number greater than 1024*1024 bytes, it returns with error (No Memory). However, if the result was less than 1024*1024, it allocates space using kmem_alloc() at line 1947 and attempts to copy the user controlled uprobe into the kernel’s probe structure. However, a user can construct a malicious fasttrap_probe_spec_t structure which has trace points (ftps_noffs) different to the kernel ones. As you can see at line 1931, noffs is completely user controlled parameter. This inconsistency between probe->ftps_noffs and uprobe->ftps_noffs can result in erroneous situations when the execution is transfered to fasttrap_add_probe() at line 1997. This function does the following:

1523 static int
1524 fasttrap_add_probe(fasttrap_probe_spec_t *pdata)
1525 {
1526     fasttrap_provider_t *provider;
1527     fasttrap_probe_t *pp;
1528     fasttrap_tracepoint_t *tp;
1529     char *name;
1530     int i, aframes, whack;
1531
1532     /*
1533      * There needs to be at least one desired trace point.
1534      */
1535     if (pdata->ftps_noffs == 0)
1536         return (EINVAL);
          ...
1567     /*
1568      * Grab the creation lock to ensure consistency between calls to
1569      * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1570      * other threads creating probes. We must drop the provider lock
1571      * before taking this lock to avoid a three-way deadlock with the
1572      * DTrace framework.
1573      */
1574     mutex_enter(&provider->ftp_cmtx);
1575
1576     if (name == NULL) {
1577         for (i = 0; i < pdata->ftps_noffs; i++) {
1578             char name_str[17];
1579
1580             (void) sprintf(name_str, "%llx",
1581                 (unsigned long long)pdata->ftps_offs[i]);
          ...
1702     return (ENOMEM);
1703 }

It is obvious that the inconsistent state can result in invalid results that are eventually leading to kernel panic. This was patched simply by adding this to the ioctl handler function:

-       if (copyin(uprobe, probe, size) != 0) {
+       if (copyin(uprobe, probe, size) != 0 ||
+           probe->ftps_noffs != noffs) {
                kmem_free(probe, size);

This IOCTL’s device is accessible at: /dev/dtrace/provider/fasttrap
The second vulnerability under this advisory was part of usr/src/uts/common/dtrace/dtrace.c file. Here is the vulnerable routine:

14522 /*ARGSUSED*/
14523 static int
14524 dtrace_ioctl_helper(int cmd, intptr_t arg, int *rv)
14525 {
14526    int rval;
14527    dof_helper_t help, *dhp = NULL;
14528
14529    switch (cmd) {
      ...
14540    case DTRACEHIOC_ADD: {
14541        dof_hdr_t *dof = dtrace_dof_copyin(arg, &rval);
14542
14543        if (dof == NULL)
14544            return (rval);
14545
14546        mutex_enter(&dtrace_lock);
14547
14548        /*
14549         * dtrace_helper_slurp() takes responsibility for the dof --
14550         * it may free it now or it may save it and free it later.
14551         */
14552        if ((rval = dtrace_helper_slurp(dof, dhp)) != -1) {
14553            *rv = rval;
14554            rval = 0;
14555        } else {
14556            rval = EINVAL;
14557        }
14558
14559        mutex_exit(&dtrace_lock);
14560        return (rval);
14561    }
      ...
14576 }


In case of a DTRACEHIOC_ADD IOCTL command to dtrace, the code at lines 14541-14560 will be executed. dof_hdr_t which is defined at usr/src/uts/common/sys/dtrace.h is the DTrace Object File (DOF) header. You can read more about that format in the aforementioned header file which has extensively detailed comments about it. Anyway, let’s move to the dtrace_helper_slurp() function which is all that this IOCTL does.

13708 static int
13709 dtrace_helper_slurp(dof_hdr_t *dof, dof_helper_t *dhp)
13710 {
        ...
13724     if ((rv = dtrace_dof_slurp(dof, vstate, NULL, &enab,
13725         dhp != NULL ? dhp->dofhp_addr : 0, B_FALSE)) != 0) {
13726         dtrace_dof_destroy(dof);
13727         return (rv);
13728     }
        ...
13802     return (gen);
13803 }

At the moment, we only care about the function being invoked at line 13724. Here is that function:

11624 /*
11625  * The dof_hdr_t passed to dtrace_dof_slurp() should be a partially validated
11626  * header:  it should be at the front of a memory region that is at least
11627  * sizeof (dof_hdr_t) in size -- and then at least dof_hdr.dofh_loadsz in
11628  * size.  It need not be validated in any other way.
11629  */
11630 static int
11631 dtrace_dof_slurp(dof_hdr_t *dof, dtrace_vstate_t *vstate, cred_t *cr,
11632     dtrace_enabling_t **enabp, uint64_t ubase, int noprobes)
11633 {
       ...
11643    /*
11644     * Check the DOF header identification bytes.  In addition to checking
11645     * valid settings, we also verify that unused bits/bytes are zeroed so
11646     * we can use them later without fear of regressing existing binaries.
11647     */
       ...


At this function numerous checks are being performed to the user controlled header. All kinds of checks, from common ones in the magic string of the file, DOF version, flag bits, sections’ alignment etc. However, it seems that they forgot a check (which I have to admit that I would have not been able to locate since I’m not familiar with DTrace Object File format). Here is the check added to the patched version:

+        if (DOF_SEC_ISLOADABLE(sec->dofs_type) &&
+            !(sec->dofs_flags & DOF_SECF_LOAD)) {
+            dtrace_dof_error(dof, "loadable section with load "
+                "flag unset");
+            return (-1);
+        }

This IOCTL routine can be reached through /dev/dtrace/helper device. I believe both vulnerabilities are fairly interesting…

UPDATE:

As Anthony Lineberry pointed out, my analysis was wrong. The problem appears when ftps_offs is greater than its actual number and thus resulting in read out of bounds. My apologies to everyone for this post.

Written by xorl

May 4, 2009 at 17:11

Posted in bugs, solaris

One Response

Subscribe to comments with RSS.

  1. […] Solaris dtrace(1M) Kernel Panic « xorl %eax, %eax […]


Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s